Skip to content

[IVDescriptors] Support reductions with minimumnum/maximumnum. #137335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 28, 2025
Merged

[IVDescriptors] Support reductions with minimumnum/maximumnum. #137335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion llvm/include/llvm/Analysis/IVDescriptors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ enum class RecurKind {
FMax, ///< FP max implemented in terms of select(cmp()).
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
FMaximumNum, ///< FP max with llvm.maximumnum semantics
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
///< loop invariant, and both x and y are integer type.
Expand Down Expand Up @@ -239,7 +241,8 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating-point min/max kind.
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum;
Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
}

/// Returns true if the recurrence kind is any min/max kind.
Expand Down
29 changes: 26 additions & 3 deletions llvm/lib/Analysis/IVDescriptors.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,10 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMax, I);
if (match(I, m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMinimumNum, I);
if (match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMaximumNum, I);
if (match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMinimum, I);
if (match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())))
Expand Down Expand Up @@ -892,10 +896,14 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
return true;
if (isa<FPMathOperator>(I) && I->hasNoNaNs() && I->hasNoSignedZeros())
return true;
// minimum and maximum intrinsics do not require nsz and nnan flags since
// NaN and signed zeroes are propagated in the intrinsic implementation.
// minimum/minnum and maximum/maxnum intrinsics do not require nsz and nnan
// flags since NaN and signed zeroes are propagated in the intrinsic
// implementation.
return match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value()));
match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())) ||
match(I,
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
Comment on lines 902 to +906
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have the misleading m_FMin and m_FMax matchers, do we not have the wrappers for the other 4?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't look like it, let me add them separately.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#137612

};
if (isIntMinMaxRecurrenceKind(Kind) ||
(HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
Expand Down Expand Up @@ -1035,6 +1043,19 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a float MINIMUM reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMaximumNum, TheLoop, FMF, RedDes, DB, AC,
DT, SE)) {
LLVM_DEBUG(dbgs() << "Found a float MAXIMUMNUM reduction PHI." << *Phi
<< "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMinimumNum, TheLoop, FMF, RedDes, DB, AC,
DT, SE)) {
LLVM_DEBUG(dbgs() << "Found a float MINIMUMNUM reduction PHI." << *Phi
<< "\n");
return true;
}

// Not a reduction of known type.
return false;
}
Expand Down Expand Up @@ -1155,6 +1176,8 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::FAnyOf:
case RecurKind::FFindLastIV:
return Instruction::FCmp;
Expand Down
13 changes: 12 additions & 1 deletion llvm/lib/Transforms/Utils/LoopUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
return Intrinsic::vector_reduce_fmaximum;
case RecurKind::FMinimum:
return Intrinsic::vector_reduce_fminimum;
case RecurKind::FMaximumNum:
return Intrinsic::vector_reduce_fmax;
case RecurKind::FMinimumNum:
return Intrinsic::vector_reduce_fmin;
}
}

Expand Down Expand Up @@ -1053,6 +1057,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
return Intrinsic::minimum;
case RecurKind::FMaximum:
return Intrinsic::maximum;
case RecurKind::FMinimumNum:
return Intrinsic::minimumnum;
case RecurKind::FMaximumNum:
return Intrinsic::maximumnum;
}
}

Expand Down Expand Up @@ -1101,7 +1109,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
Value *Right) {
Type *Ty = Left->getType();
if (Ty->isIntOrIntVectorTy() ||
(RK == RecurKind::FMinimum || RK == RecurKind::FMaximum)) {
(RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
// TODO: Add float minnum/maxnum support when FMF nnan is set.
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
Expand Down Expand Up @@ -1320,6 +1329,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::FMin:
case RecurKind::FMinimum:
case RecurKind::FMaximum:
case RecurKind::FMinimumNum:
case RecurKind::FMaximumNum:
return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
case RecurKind::FMulAdd:
case RecurKind::FAdd:
Expand Down
10 changes: 9 additions & 1 deletion llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21774,7 +21774,9 @@ class HorizontalReduction {
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum: {
case RecurKind::FMinimum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum: {
Intrinsic::ID Id = llvm::getMinMaxReductionIntrinsicOp(Kind);
return Builder.CreateBinaryIntrinsic(Id, LHS, RHS);
}
Expand Down Expand Up @@ -23086,6 +23088,8 @@ class HorizontalReduction {
case RecurKind::FAnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
Expand Down Expand Up @@ -23220,6 +23224,8 @@ class HorizontalReduction {
case RecurKind::FAnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
Expand Down Expand Up @@ -23319,6 +23325,8 @@ class HorizontalReduction {
case RecurKind::FAnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for reused scalars.");
}
Expand Down
Loading
Loading