[VPlan] Introduce VPWidenIntrinsicRecipe to separate from libcall. #110486
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesThis patch splits off intrinsic hanlding to a new This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR. Patch is 28.02 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/110486.diff 12 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 08e78cb49c69fc..46482ed9e98ae7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4367,7 +4367,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
[](const auto *R) { return Instruction::Store; })
.Case<VPWidenLoadRecipe>(
[](const auto *R) { return Instruction::Load; })
- .Case<VPWidenCallRecipe>(
+ .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
[](const auto *R) { return Instruction::Call; })
.Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
VPWidenCastRecipe>(
@@ -4392,11 +4392,17 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
OS << "):";
if (Opcode == Instruction::Call) {
auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
- Function *CalledFn =
- WidenCall ? WidenCall->getCalledScalarFunction()
- : cast<Function>(R->getOperand(R->getNumOperands() - 1)
- ->getLiveInIRValue());
- OS << " call to " << CalledFn->getName();
+ StringRef Name = "";
+ if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R))
+ Name = Int->getIntrinsicName();
+ else {
+ Function *CalledFn =
+ WidenCall ? WidenCall->getCalledScalarFunction()
+ : cast<Function>(R->getOperand(R->getNumOperands() - 1)
+ ->getLiveInIRValue());
+ Name = CalledFn->getName();
+ }
+ OS << " call to " << Name;
} else
OS << " " << Instruction::getOpcodeName(Opcode);
reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
@@ -4447,6 +4453,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPWidenCanonicalIVSC:
case VPDef::VPWidenCastSC:
case VPDef::VPWidenGEPSC:
+ case VPDef::VPWidenIntrinsicSC:
case VPDef::VPWidenSC:
case VPDef::VPWidenSelectSC:
case VPDef::VPBlendSC:
@@ -8266,7 +8273,7 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi,
return new VPBlendRecipe(Phi, OperandsWithMask);
}
-VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
+VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
ArrayRef<VPValue *> Operands,
VFRange &Range) {
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
@@ -8297,8 +8304,9 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
},
Range);
if (ShouldUseVectorIntrinsic)
- return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID,
- CI->getDebugLoc());
+ return new VPWidenIntrinsicRecipe(CI, ID,
+ make_range(Ops.begin(), Ops.end() - 1),
+ CI->getType(), CI->getDebugLoc());
Function *Variant = nullptr;
std::optional<unsigned> MaskPos;
@@ -8350,9 +8358,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()),
- Intrinsic::not_intrinsic, CI->getDebugLoc(),
- Variant);
+ return new VPWidenCallRecipe(
+ CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc());
}
return nullptr;
@@ -9218,7 +9225,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
"Expected instruction to be a call to the llvm.fmuladd intrinsic");
assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
- isa<VPWidenCallRecipe>(CurrentLink)) &&
+ isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
CurrentLink->getOperand(2) == PreviousLink &&
"expected a call where the previous link is the added operand");
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 02ef6f7b6fb982..5d4a3b555981ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -93,9 +93,9 @@ class VPRecipeBuilder {
VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands);
/// Handle call instructions. If \p CI can be widened for \p Range.Start,
- /// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same
- /// decision from \p Range.Start to \p Range.End.
- VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
+ /// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be
+ /// decreased to ensure same decision from \p Range.Start to \p Range.End.
+ VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
VFRange &Range);
/// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c4567362eaffc7..b7b2d000eb77ea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -886,6 +886,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenCanonicalIVSC:
case VPRecipeBase::VPWidenCastSC:
case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPWidenIntrinsicSC:
case VPRecipeBase::VPWidenSC:
case VPRecipeBase::VPWidenEVLSC:
case VPRecipeBase::VPWidenSelectSC:
@@ -1608,24 +1609,63 @@ class VPScalarCastRecipe : public VPSingleDefRecipe {
}
};
-/// A recipe for widening Call instructions.
-class VPWidenCallRecipe : public VPSingleDefRecipe {
- /// ID of the vector intrinsic to call when widening the call. If set the
- /// Intrinsic::not_intrinsic, a library call will be used instead.
+/// A recipe for widening vector intrinsics.
+class VPWidenIntrinsicRecipe : public VPSingleDefRecipe {
+ /// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
- /// If this recipe represents a library call, Variant stores a pointer to
- /// the chosen function. There is a 1:1 mapping between a given VF and the
- /// chosen vectorized variant, so there will be a different vplan for each
- /// VF with a valid variant.
+
+ /// Scalar type of the result produced by the intrinsic.
+ Type *ResultTy;
+
+public:
+ template <typename IterT>
+ VPWidenIntrinsicRecipe(Value *UV, Intrinsic::ID VectorIntrinsicID,
+ iterator_range<IterT> CallArguments, Type *Ty,
+ DebugLoc DL = {})
+ : VPSingleDefRecipe(VPDef::VPWidenIntrinsicSC, CallArguments, UV, DL),
+ VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {}
+
+ ~VPWidenIntrinsicRecipe() override = default;
+
+ VPWidenIntrinsicRecipe *clone() override {
+ return new VPWidenIntrinsicRecipe(getUnderlyingValue(), VectorIntrinsicID,
+ operands(), ResultTy, getDebugLoc());
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
+
+ /// Produce a widened version of the vector intrinsic.
+ void execute(VPTransformState &State) override;
+
+ /// Return the cost of this vector intrinsic.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+ Type *getResultTy() const { return ResultTy; }
+
+ /// Return to name of the intrinsic as string.
+ StringRef getIntrinsicName() const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
+/// A recipe for widening Call instructions using library calls.
+class VPWidenCallRecipe : public VPSingleDefRecipe {
+ /// Variant stores a pointer to the chosen function. There is a 1:1 mapping
+ /// between a given VF and the chosen vectorized variant, so there will be a
+ /// different VPlan for each VF with a valid variant.
Function *Variant;
public:
template <typename IterT>
- VPWidenCallRecipe(Value *UV, iterator_range<IterT> CallArguments,
- Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {},
- Function *Variant = nullptr)
+ VPWidenCallRecipe(Value *UV, Function *Variant,
+ iterator_range<IterT> CallArguments, DebugLoc DL = {})
: VPSingleDefRecipe(VPDef::VPWidenCallSC, CallArguments, UV, DL),
- VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {
+ Variant(Variant) {
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1634,8 +1674,8 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
~VPWidenCallRecipe() override = default;
VPWidenCallRecipe *clone() override {
- return new VPWidenCallRecipe(getUnderlyingValue(), operands(),
- VectorIntrinsicID, getDebugLoc(), Variant);
+ return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 277df0637372d8..24f34e43a6e84b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -268,6 +268,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe,
VPWidenSelectRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
+ .Case<VPWidenIntrinsicRecipe>(
+ [](const VPWidenIntrinsicRecipe *R) { return R->getResultTy(); })
.Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) {
// TODO: Use info from interleave group.
return V->getUnderlyingValue()->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f8b0a400a31d7d..faf6931e066556 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -79,6 +79,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
return !cast<VPWidenCallRecipe>(this)
->getCalledScalarFunction()
->onlyReadsMemory();
+ case VPWidenIntrinsicSC:
+ return false;
case VPBranchOnMaskSC:
case VPScalarIVStepsSC:
case VPPredInstPHISC:
@@ -120,6 +122,8 @@ bool VPRecipeBase::mayReadFromMemory() const {
return !cast<VPWidenCallRecipe>(this)
->getCalledScalarFunction()
->onlyWritesMemory();
+ case VPWidenIntrinsicSC:
+ return true;
case VPBranchOnMaskSC:
case VPPredInstPHISC:
case VPScalarIVStepsSC:
@@ -161,6 +165,8 @@ bool VPRecipeBase::mayHaveSideEffects() const {
Function *Fn = cast<VPWidenCallRecipe>(this)->getCalledScalarFunction();
return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn();
}
+ case VPWidenIntrinsicSC:
+ return false;
case VPBlendSC:
case VPReductionEVLSC:
case VPReductionSC:
@@ -879,61 +885,31 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
void VPWidenCallRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
- Function *CalledScalarFn = getCalledScalarFunction();
- assert(!isDbgInfoIntrinsic(CalledScalarFn->getIntrinsicID()) &&
- "DbgInfoIntrinsic should have been dropped during VPlan construction");
State.setDebugLocFrom(getDebugLoc());
- bool UseIntrinsic = VectorIntrinsicID != Intrinsic::not_intrinsic;
- FunctionType *VFTy = nullptr;
- if (Variant)
- VFTy = Variant->getFunctionType();
- SmallVector<Type *, 2> TysForDecl;
+ FunctionType *VFTy = Variant->getFunctionType();
// Add return type if intrinsic is overloaded on it.
- if (UseIntrinsic &&
- isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1))
- TysForDecl.push_back(VectorType::get(
- CalledScalarFn->getReturnType()->getScalarType(), State.VF));
SmallVector<Value *, 4> Args;
for (const auto &I : enumerate(arg_operands())) {
- // Some intrinsics have a scalar argument - don't replace it with a
- // vector.
Value *Arg;
- if (UseIntrinsic &&
- isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
- Arg = State.get(I.value(), VPLane(0));
// Some vectorized function variants may also take a scalar argument,
// e.g. linear parameters for pointers. This needs to be the scalar value
// from the start of the respective part when interleaving.
- else if (VFTy && !VFTy->getParamType(I.index())->isVectorTy())
+ if (!VFTy->getParamType(I.index())->isVectorTy())
Arg = State.get(I.value(), VPLane(0));
else
- Arg = State.get(I.value());
- if (UseIntrinsic &&
- isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
- TysForDecl.push_back(Arg->getType());
+ Arg = State.get(I.value(), onlyFirstLaneUsed(I.value()));
Args.push_back(Arg);
}
- Function *VectorF;
- if (UseIntrinsic) {
- // Use vector version of the intrinsic.
- Module *M = State.Builder.GetInsertBlock()->getModule();
- VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF && "Can't retrieve vector intrinsic.");
- } else {
-#ifndef NDEBUG
- assert(Variant != nullptr && "Can't create vector function.");
-#endif
- VectorF = Variant;
- }
+ assert(Variant != nullptr && "Can't create vector function.");
- auto *CI = cast_or_null<CallInst>(getUnderlyingInstr());
+ auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
SmallVector<OperandBundleDef, 1> OpBundles;
if (CI)
CI->getOperandBundlesAsDefs(OpBundles);
- CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+ CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
if (isa<FPMathOperator>(V))
V->copyFastMathFlags(CI);
@@ -946,12 +922,84 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- if (Variant) {
- return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(),
- Variant->getFunctionType()->params(),
- CostKind);
+ return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(),
+ Variant->getFunctionType()->params(),
+ CostKind);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-CALL ";
+
+ Function *CalledFn = getCalledScalarFunction();
+ if (CalledFn->getReturnType()->isVoidTy())
+ O << "void ";
+ else {
+ printAsOperand(O, SlotTracker);
+ O << " = ";
+ }
+
+ O << "call @" << CalledFn->getName() << "(";
+ interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
+ O << ")";
+
+ O << " (using library function";
+ if (Variant->hasName())
+ O << ": " << Variant->getName();
+ O << ")";
+}
+#endif
+
+void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
+ assert(State.VF.isVector() && "not widening");
+ State.setDebugLocFrom(getDebugLoc());
+
+ SmallVector<Type *, 2> TysForDecl;
+ // Add return type if intrinsic is overloaded on it.
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1))
+ TysForDecl.push_back(VectorType::get(getResultTy(), State.VF));
+ SmallVector<Value *, 4> Args;
+ for (const auto &I : enumerate(operands())) {
+ // Some intrinsics have a scalar argument - don't replace it with a
+ // vector.
+ Value *Arg;
+ if (isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
+ Arg = State.get(I.value(), VPLane(0));
+ else
+ Arg = State.get(I.value(), onlyFirstLaneUsed(I.value()));
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
+ TysForDecl.push_back(Arg->getType());
+ Args.push_back(Arg);
}
+ // Use vector version of the intrinsic.
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ Function *VectorF =
+ Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF && "Can't retrieve vector intrinsic.");
+
+ auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ if (CI)
+ CI->getOperandBundlesAsDefs(OpBundles);
+
+ CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+
+ if (isa<FPMathOperator>(V))
+ V->copyFastMathFlags(CI);
+
+ if (!V->getType()->isVoidTy())
+ State.set(this, V);
+ State.addMetadata(V, CI);
+}
+
+InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+
FastMathFlags FMF;
// TODO: Manage flags via VPRecipeWithIRFlags.
if (auto *FPMO = dyn_cast_or_null<FPMathOperator>(getUnderlyingValue()))
@@ -990,33 +1038,27 @@ InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
}
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "WIDEN-CALL ";
+StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
+ return Intrinsic::getBaseName(VectorIntrinsicID);
+}
- Function *CalledFn = getCalledScalarFunction();
- if (CalledFn->getReturnType()->isVoidTy())
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-INTRINSIC ";
+ if (ResultTy->isVoidTy()) {
O << "void ";
- else {
+ } else {
printAsOperand(O, SlotTracker);
O << " = ";
}
- O << "call @" << CalledFn->getName() << "(";
- interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) {
+ O << getIntrinsicName() << "(";
+
+ interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) {
Op->printAsOperand(O, SlotTracker);
});
O << ")";
-
- if (VectorIntrinsicID)
- O << " (using vector intrinsic)";
- else {
- O << " (using library function";
- if (Variant->hasName())
- O << ": " << Variant->getName();
- O << ")";
- }
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a878613c4ba483..8ae4291de9e9a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -82,9 +82,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
- NewRecipe = new VPWidenCallRecipe(
- CI, Ingredient.operands(), getVectorIntrinsicIDForCall(CI, &TLI),
- CI->getDebugLoc());
+ NewRecipe = new VPWidenIntrinsicRecipe(
+ CI, getVectorIntrinsicIDForCall(CI, &TLI),
+ make_range(Ingredient.op_begin(), Ingredient.op_end() - 1),
+ CI->getType(), CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
N...
[truncated]
|
fhahn
added a commit
to fhahn/llvm-project
that referenced
this pull request
Sep 30, 2024
Use VPWidenIntrinsicRecipe (llvm#110486) to create vp.select intrinsics. This potentially offers an alternative to duplicating EVL recipes for all existing recipes. There are some recipes that will need duplicates (at least at the moment), due to extra code-gen needs (e.g. widening loads and stores). But in cases the intrinsic can directly be used, creating the widened intrinsic directly would reduce the need to duplicate some recipes. NOTE: this PR contains the changes from llvm#110486) The relevant changes are in 47135542e2ada3f21b215bf237d8442a56c8456c.
This was referenced Sep 30, 2024
fhahn
force-pushed
the
vplan-widen-intrinsic
branch
from
f9c9f33 to
bba6b71
Compare
This patch splits off intrinsic hanlding to a new VPWidenIntrinsicRecipe. VPWidenIntrinsicRecipes only need access to the intrinsic ID to widen and the scalar result type (in case the intrinsic is overloaded on the result type). It does not need access to an underlying IR call instruction or function. This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR.
fhahn
added a commit
to fhahn/llvm-project
that referenced
this pull request
Oct 1, 2024
Use VPWidenIntrinsicRecipe (llvm#110486) to create vp.select intrinsics. This potentially offers an alternative to duplicating EVL recipes for all existing recipes. There are some recipes that will need duplicates (at least at the moment), due to extra code-gen needs (e.g. widening loads and stores). But in cases the intrinsic can directly be used, creating the widened intrinsic directly would reduce the need to duplicate some recipes. NOTE: this PR contains the changes from llvm#110486) The relevant changes are in 47135542e2ada3f21b215bf237d8442a56c8456c.
fhahn
force-pushed
the
vplan-widen-intrinsic
branch
from
bba6b71 to
8ae59c7
Compare
arcbbb
reviewed
Oct 3, 2024
| @@ -79,6 +79,8 @@ bool VPRecipeBase::mayWriteToMemory() const { | |||
| return !cast<VPWidenCallRecipe>(this) | |||
| ->getCalledScalarFunction() | |||
| ->onlyReadsMemory(); | |||
| case VPWidenIntrinsicSC: | |||
| return false; | |||
There was a problem hiding this comment.
This might be incorrect for memaccesses intrinsics. Need somehow to restrict the class for using with such intrinsics
There was a problem hiding this comment.
Yep, updated to store read/write/sideeffcts in recipe.
Comment on lines
125
to
126
| case VPWidenIntrinsicSC: | ||
| return true; |
There was a problem hiding this comment.
Yep, updated to store read/write/sideeffcts in recipe.
Comment on lines
168
to
169
| case VPWidenIntrinsicSC: | ||
| return false; |
There was a problem hiding this comment.
Yep, updated to store read/write/sideeffcts in recipe.
| return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID, | ||
| CI->getDebugLoc()); | ||
| return new VPWidenIntrinsicRecipe(*CI, ID, | ||
| make_range(Ops.begin(), Ops.end() - 1), |
There was a problem hiding this comment.
Maybe move Ops.push_back(Operands.back()); from line 8305 to line 8319 to avoid -1 here?
| public: | ||
| template <typename IterT> | ||
| VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, | ||
| iterator_range<IterT> CallArguments, Type *Ty, |
There was a problem hiding this comment.
What about ArrayRef here instead of iterator_range?
Mel-Chen
reviewed
Oct 7, 2024
| @@ -79,6 +79,9 @@ bool VPRecipeBase::mayWriteToMemory() const { | |||
| return !cast<VPWidenCallRecipe>(this) | |||
| ->getCalledScalarFunction() | |||
| ->onlyReadsMemory(); | |||
| case VPWidenIntrinsicSC: | |||
| return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory(); | |||
| ; | |||
| CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles); | ||
|
|
||
| if (isa<FPMathOperator>(V)) | ||
| V->copyFastMathFlags(CI); |
There was a problem hiding this comment.
Is it possible for CI to be nullptr?
There was a problem hiding this comment.
Left over from an older version after a rebase, should be setFlags(V) as on current main, fixed, thanks!
Comment on lines
8360
to
8361
| return new VPWidenCallRecipe( | ||
| CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc()); |
There was a problem hiding this comment.
Suggested change
| return new VPWidenCallRecipe( | |
| CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc()); | |
| return new VPWidenCallRecipe( | |
| CI, Variant, Ops, CI->getDebugLoc()); |
fhahn
added a commit
that referenced
this pull request
Oct 15, 2024
Use VPWidenIntrinsicRecipe (#110486) to create vp.select intrinsics. This potentially offers an alternative to duplicating EVL recipes for all existing recipes. There are some recipes that will need duplicates (at least at the moment), due to extra code-gen needs (e.g. widening loads and stores). But in cases the intrinsic can directly be used, creating the widened intrinsic directly would reduce the need to duplicate some recipes. PR: #110489
DanielCChen
pushed a commit
to DanielCChen/llvm-project
that referenced
this pull request
Oct 16, 2024
Use VPWidenIntrinsicRecipe (llvm#110486) to create vp.select intrinsics. This potentially offers an alternative to duplicating EVL recipes for all existing recipes. There are some recipes that will need duplicates (at least at the moment), due to extra code-gen needs (e.g. widening loads and stores). But in cases the intrinsic can directly be used, creating the widened intrinsic directly would reduce the need to duplicate some recipes. PR: llvm#110489
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This patch splits off intrinsic hanlding to a new
VPWidenIntrinsicRecipe. VPWidenIntrinsicRecipes only need access to the intrinsic ID to widen and the scalar result type (in case the intrinsic is overloaded on the result type). It does not need access to an underlying IR call instruction or function.
This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR.