@@ -1892,10 +1892,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
18921892 return ;
18931893 }
18941894
1895+ // Emit the code with the fully general case.
1896+ llvm::Value *CondV;
1897+ {
1898+ ApplyDebugLocation DL (*this , Cond);
1899+ CondV = EvaluateExprAsBool (Cond);
1900+ }
1901+
1902+ llvm::MDNode *Weights = nullptr ;
1903+ llvm::MDNode *Unpredictable = nullptr ;
1904+
18951905 // If the branch has a condition wrapped by __builtin_unpredictable,
18961906 // create metadata that specifies that the branch is unpredictable.
18971907 // Don't bother if not optimizing because that metadata would not be used.
1898- llvm::MDNode *Unpredictable = nullptr ;
18991908 auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts ());
19001909 if (Call && CGM.getCodeGenOpts ().OptimizationLevel != 0 ) {
19011910 auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl ());
@@ -1905,18 +1914,17 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
19051914 }
19061915 }
19071916
1908- llvm::MDNode *Weights = createBranchWeights (LH);
1909- if (!Weights) {
1917+ // If there is a Likelihood knowledge for the cond, lower it.
1918+ // Note that if not optimizing this won't emit anything.
1919+ llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic (CondV, LH);
1920+ if (CondV != NewCondV)
1921+ CondV = NewCondV;
1922+ else {
1923+ // Otherwise, lower profile counts. Note that we do this even at -O0.
19101924 uint64_t CurrentCount = std::max (getCurrentProfileCount (), TrueCount);
19111925 Weights = createProfileWeights (TrueCount, CurrentCount - TrueCount);
19121926 }
19131927
1914- // Emit the code with the fully general case.
1915- llvm::Value *CondV;
1916- {
1917- ApplyDebugLocation DL (*this , Cond);
1918- CondV = EvaluateExprAsBool (Cond);
1919- }
19201928 Builder.CreateCondBr (CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
19211929}
19221930
@@ -2808,35 +2816,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
28082816 return llvm::DebugLoc ();
28092817}
28102818
2811- static Optional<std::pair<uint32_t , uint32_t >>
2812- getLikelihoodWeights (Stmt::Likelihood LH) {
2819+ llvm::Value *
2820+ CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic (llvm::Value *Cond,
2821+ Stmt::Likelihood LH) {
28132822 switch (LH) {
2814- case Stmt::LH_Unlikely:
2815- return std::pair<uint32_t , uint32_t >(llvm::UnlikelyBranchWeight,
2816- llvm::LikelyBranchWeight);
28172823 case Stmt::LH_None:
2818- return None ;
2824+ return Cond ;
28192825 case Stmt::LH_Likely:
2820- return std::pair<uint32_t , uint32_t >(llvm::LikelyBranchWeight,
2821- llvm::UnlikelyBranchWeight);
2826+ case Stmt::LH_Unlikely:
2827+ // Don't generate llvm.expect on -O0 as the backend won't use it for
2828+ // anything.
2829+ if (CGM.getCodeGenOpts ().OptimizationLevel == 0 )
2830+ return Cond;
2831+ llvm::Type *CondTy = Cond->getType ();
2832+ assert (CondTy->isIntegerTy (1 ) && " expecting condition to be a boolean"
2833+ llvm::Function *FnExpect =
2834+ CGM.getIntrinsic (llvm::Intrinsic::expect, CondTy);
2835+ llvm::Value *ExpectedValueOfCond =
2836+ llvm::ConstantInt::getBool (CondTy, LH == Stmt::LH_Likely);
2837+ return Builder.CreateCall (FnExpect, {Cond, ExpectedValueOfCond},
2838+ Cond->getName () + " .expval"
28222839 }
28232840 llvm_unreachable (" Unknown Likelihood"
28242841}
2825-
2826- llvm::MDNode *CodeGenFunction::createBranchWeights (Stmt::Likelihood LH) const {
2827- Optional<std::pair<uint32_t , uint32_t >> LHW = getLikelihoodWeights (LH);
2828- if (!LHW)
2829- return nullptr ;
2830-
2831- llvm::MDBuilder MDHelper (CGM.getLLVMContext ());
2832- return MDHelper.createBranchWeights (LHW->first , LHW->second );
2833- }
2834-
2835- llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop (
2836- const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
2837- llvm::MDNode *Weights = createProfileWeightsForLoop (Cond, LoopCount);
2838- if (!Weights && CGM.getCodeGenOpts ().OptimizationLevel )
2839- Weights = createBranchWeights (Stmt::getLikelihood (Body));
2840-
2841- return Weights;
2842- }
0 commit comments