Skip to content

Commit bba6b71

Browse files
fhahnfhahn
committed
[VPlan] Introduce VPWidenIntrinsicRecipe to separate from libcall.
This patch splits off intrinsic hanlding to a new VPWidenIntrinsicRecipe. VPWidenIntrinsicRecipes only need access to the intrinsic ID to widen and the scalar result type (in case the intrinsic is overloaded on the result type). It does not need access to an underlying IR call instruction or function. This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR.
1 parent 007780f commit bba6b71

File tree

12 files changed

+197
-101
lines changed

12 files changed

+197
-101
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+20-13
Original file line numberDiff line numberDiff line change
@@ -4367,7 +4367,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
43674367
[](const auto *R) { return Instruction::Store; })
43684368
.Case<VPWidenLoadRecipe>(
43694369
[](const auto *R) { return Instruction::Load; })
4370-
.Case<VPWidenCallRecipe>(
4370+
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
43714371
[](const auto *R) { return Instruction::Call; })
43724372
.Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
43734373
VPWidenCastRecipe>(
@@ -4392,11 +4392,17 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
43924392
OS << "):";
43934393
if (Opcode == Instruction::Call) {
43944394
auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
4395-
Function *CalledFn =
4396-
WidenCall ? WidenCall->getCalledScalarFunction()
4397-
: cast<Function>(R->getOperand(R->getNumOperands() - 1)
4398-
->getLiveInIRValue());
4399-
OS << " call to " << CalledFn->getName();
4395+
StringRef Name = "";
4396+
if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
4397+
Name = Int->getIntrinsicName();
4398+
} else {
4399+
Function *CalledFn =
4400+
WidenCall ? WidenCall->getCalledScalarFunction()
4401+
: cast<Function>(R->getOperand(R->getNumOperands() - 1)
4402+
->getLiveInIRValue());
4403+
Name = CalledFn->getName();
4404+
}
4405+
OS << " call to " << Name;
44004406
} else
44014407
OS << " " << Instruction::getOpcodeName(Opcode);
44024408
reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
@@ -4447,6 +4453,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
44474453
case VPDef::VPWidenCanonicalIVSC:
44484454
case VPDef::VPWidenCastSC:
44494455
case VPDef::VPWidenGEPSC:
4456+
case VPDef::VPWidenIntrinsicSC:
44504457
case VPDef::VPWidenSC:
44514458
case VPDef::VPWidenSelectSC:
44524459
case VPDef::VPBlendSC:
@@ -8266,7 +8273,7 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi,
82668273
return new VPBlendRecipe(Phi, OperandsWithMask);
82678274
}
82688275

8269-
VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
8276+
VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
82708277
ArrayRef<VPValue *> Operands,
82718278
VFRange &Range) {
82728279
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
@@ -8297,8 +8304,9 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
82978304
},
82988305
Range);
82998306
if (ShouldUseVectorIntrinsic)
8300-
return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID,
8301-
CI->getDebugLoc());
8307+
return new VPWidenIntrinsicRecipe(CI, ID,
8308+
make_range(Ops.begin(), Ops.end() - 1),
8309+
CI->getType(), CI->getDebugLoc());
83028310

83038311
Function *Variant = nullptr;
83048312
std::optional<unsigned> MaskPos;
@@ -8350,9 +8358,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
83508358
Ops.insert(Ops.begin() + *MaskPos, Mask);
83518359
}
83528360

8353-
return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()),
8354-
Intrinsic::not_intrinsic, CI->getDebugLoc(),
8355-
Variant);
8361+
return new VPWidenCallRecipe(
8362+
CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc());
83568363
}
83578364

83588365
return nullptr;
@@ -9218,7 +9225,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92189225
RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
92199226
"Expected instruction to be a call to the llvm.fmuladd intrinsic");
92209227
assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
9221-
isa<VPWidenCallRecipe>(CurrentLink)) &&
9228+
isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
92229229
CurrentLink->getOperand(2) == PreviousLink &&
92239230
"expected a call where the previous link is the added operand");
92249231

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

+3-3
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,9 @@ class VPRecipeBuilder {
9393
VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands);
9494

9595
/// Handle call instructions. If \p CI can be widened for \p Range.Start,
96-
/// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same
97-
/// decision from \p Range.Start to \p Range.End.
98-
VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
96+
/// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be
97+
/// decreased to ensure same decision from \p Range.Start to \p Range.End.
98+
VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
9999
VFRange &Range);
100100

101101
/// Check if \p I has an opcode that can be widened and return a VPWidenRecipe

‎llvm/lib/Transforms/Vectorize/VPlan.h

+55-14
Original file line numberDiff line numberDiff line change
@@ -886,6 +886,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
886886
case VPRecipeBase::VPWidenCanonicalIVSC:
887887
case VPRecipeBase::VPWidenCastSC:
888888
case VPRecipeBase::VPWidenGEPSC:
889+
case VPRecipeBase::VPWidenIntrinsicSC:
889890
case VPRecipeBase::VPWidenSC:
890891
case VPRecipeBase::VPWidenEVLSC:
891892
case VPRecipeBase::VPWidenSelectSC:
@@ -1608,25 +1609,65 @@ class VPScalarCastRecipe : public VPSingleDefRecipe {
16081609
}
16091610
};
16101611

1611-
/// A recipe for widening Call instructions.
1612-
class VPWidenCallRecipe : public VPRecipeWithIRFlags {
1613-
/// ID of the vector intrinsic to call when widening the call. If set the
1614-
/// Intrinsic::not_intrinsic, a library call will be used instead.
1612+
/// A recipe for widening vector intrinsics.
1613+
class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
1614+
/// ID of the vector intrinsic to widen.
16151615
Intrinsic::ID VectorIntrinsicID;
1616-
/// If this recipe represents a library call, Variant stores a pointer to
1617-
/// the chosen function. There is a 1:1 mapping between a given VF and the
1618-
/// chosen vectorized variant, so there will be a different vplan for each
1619-
/// VF with a valid variant.
1616+
1617+
/// Scalar type of the result produced by the intrinsic.
1618+
Type *ResultTy;
1619+
1620+
public:
1621+
template <typename IterT>
1622+
VPWidenIntrinsicRecipe(Value *UV, Intrinsic::ID VectorIntrinsicID,
1623+
iterator_range<IterT> CallArguments, Type *Ty,
1624+
DebugLoc DL = {})
1625+
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments,
1626+
*cast<Instruction>(UV)),
1627+
VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {}
1628+
1629+
~VPWidenIntrinsicRecipe() override = default;
1630+
1631+
VPWidenIntrinsicRecipe *clone() override {
1632+
return new VPWidenIntrinsicRecipe(getUnderlyingValue(), VectorIntrinsicID,
1633+
operands(), ResultTy, getDebugLoc());
1634+
}
1635+
1636+
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
1637+
1638+
/// Produce a widened version of the vector intrinsic.
1639+
void execute(VPTransformState &State) override;
1640+
1641+
/// Return the cost of this vector intrinsic.
1642+
InstructionCost computeCost(ElementCount VF,
1643+
VPCostContext &Ctx) const override;
1644+
1645+
Type *getResultTy() const { return ResultTy; }
1646+
1647+
/// Return to name of the intrinsic as string.
1648+
StringRef getIntrinsicName() const;
1649+
1650+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1651+
/// Print the recipe.
1652+
void print(raw_ostream &O, const Twine &Indent,
1653+
VPSlotTracker &SlotTracker) const override;
1654+
#endif
1655+
};
1656+
1657+
/// A recipe for widening Call instructions using library calls.
1658+
class VPWidenCallRecipe : public VPRecipeWithIRFlags {
1659+
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
1660+
/// between a given VF and the chosen vectorized variant, so there will be a
1661+
/// different VPlan for each VF with a valid variant.
16201662
Function *Variant;
16211663

16221664
public:
16231665
template <typename IterT>
1624-
VPWidenCallRecipe(Value *UV, iterator_range<IterT> CallArguments,
1625-
Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {},
1626-
Function *Variant = nullptr)
1666+
VPWidenCallRecipe(Value *UV, Function *Variant,
1667+
iterator_range<IterT> CallArguments, DebugLoc DL = {})
16271668
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
16281669
*cast<Instruction>(UV)),
1629-
VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {
1670+
Variant(Variant) {
16301671
assert(
16311672
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
16321673
"last operand must be the called function");
@@ -1635,8 +1676,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags {
16351676
~VPWidenCallRecipe() override = default;
16361677

16371678
VPWidenCallRecipe *clone() override {
1638-
return new VPWidenCallRecipe(getUnderlyingValue(), operands(),
1639-
VectorIntrinsicID, getDebugLoc(), Variant);
1679+
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
1680+
getDebugLoc());
16401681
}
16411682

16421683
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
268268
VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe,
269269
VPWidenSelectRecipe>(
270270
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
271+
.Case<VPWidenIntrinsicRecipe>(
272+
[](const VPWidenIntrinsicRecipe *R) { return R->getResultTy(); })
271273
.Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) {
272274
// TODO: Use info from interleave group.
273275
return V->getUnderlyingValue()->getType();

0 commit comments

Comments
 (0)