Skip to content

Commit 7487894

Browse files
rajatdrajatd
committed
Inlining in jitted loop bodies
1 parent cc613d8 commit 7487894

19 files changed

+210
-133
lines changed

‎lib/Backend/BackwardPass.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6497,7 +6497,7 @@ BackwardPass::ProcessInlineeStart(IR::Instr* inlineeStart)
64976497
inlineeStart->IterateMetaArgs([&](IR::Instr* metaArg)
64986498
{
64996499
if (i == Js::Constants::InlineeMetaArgIndex_ArgumentsObject &&
6500-
inlineeStart->m_func->GetHasArgumentObject())
6500+
inlineeStart->m_func->GetJnFunction()->GetUsesArgumentsObject())
65016501
{
65026502
Assert(!inlineeStart->m_func->GetHasUnoptimizedArgumentsAcccess());
65036503
// Do not remove arguments object meta arg if there is a reference to arguments object

‎lib/Backend/Encoder.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ void Encoder::RecordInlineeFrame(Func* inlinee, uint32 currentOffset)
595595
{
596596
// The only restriction for not supporting loop bodies is that inlinee frame map is created on FunctionEntryPointInfo & not
597597
// the base class EntryPointInfo.
598-
if (!this->m_func->IsLoopBody() && !this->m_func->IsSimpleJit())
598+
if (!(this->m_func->IsLoopBody() && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->m_func)) && !this->m_func->IsSimpleJit())
599599
{
600600
InlineeFrameRecord* record = nullptr;
601601
if (inlinee->frameInfo && inlinee->m_hasInlineArgsOpt)

‎lib/Backend/Func.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ Func::Func(JitArenaAllocator *alloc, CodeGenWorkItem* workItem, const Js::Functi
7676
hasInlinee(false),
7777
thisOrParentInlinerHasArguments(false),
7878
hasStackArgs(false),
79-
hasArgumentObject(false),
8079
hasUnoptimizedArgumentsAcccess(false),
8180
hasApplyTargetInlining(false),
8281
hasImplicitCalls(false),
@@ -148,7 +147,8 @@ Func::Func(JitArenaAllocator *alloc, CodeGenWorkItem* workItem, const Js::Functi
148147
// as determined by the bytecode generator.
149148
SetHasStackArgs(true);
150149
}
151-
if (doStackNestedFunc && m_jnFunction->GetNestedCount() != 0)
150+
if (doStackNestedFunc && m_jnFunction->GetNestedCount() != 0 &&
151+
this->GetTopFunc()->m_workItem->Type() != JsLoopBodyWorkItemType) // make sure none of the functions inlined in a jitted loop body allocate nested functions on the stack
152152
{
153153
Assert(!(this->IsJitInDebugMode() && !m_jnFunction->GetUtf8SourceInfo()->GetIsLibraryCode()));
154154
stackNestedFunc = true;
@@ -376,7 +376,7 @@ Func::Codegen()
376376

377377
BEGIN_CODEGEN_PHASE(this, Js::InlinePhase);
378378

379-
InliningHeuristics heuristics(this->GetJnFunction());
379+
InliningHeuristics heuristics(this->GetJnFunction(), this->IsLoopBody());
380380
Inline inliner(this, heuristics);
381381
inliner.Optimize();
382382

‎lib/Backend/Func.h

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,6 @@ static const unsigned __int64 c_debugFillPattern8 = 0xcececececececece;
516516
bool hasBailout: 1;
517517
bool hasBailoutInEHRegion : 1;
518518
bool hasStackArgs: 1;
519-
bool hasArgumentObject : 1;
520519
bool hasUnoptimizedArgumentsAcccess : 1; // True if there are any arguments access beyond the simple case of this.apply pattern
521520
bool m_canDoInlineArgsOpt : 1;
522521
bool hasApplyTargetInlining:1;
@@ -580,9 +579,6 @@ static const unsigned __int64 c_debugFillPattern8 = 0xcececececececece;
580579
bool GetHasStackArgs() const { return this->hasStackArgs;}
581580
void SetHasStackArgs(bool has) { this->hasStackArgs = has;}
582581

583-
bool GetHasArgumentObject() const { return this->hasArgumentObject;}
584-
void SetHasArgumentObject() { this->hasArgumentObject = true;}
585-
586582
bool GetHasUnoptimizedArgumentsAcccess() const { return this->hasUnoptimizedArgumentsAcccess; }
587583
void SetHasUnoptimizedArgumentsAccess(bool args)
588584
{

‎lib/Backend/IRBuilder.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1483,7 +1483,6 @@ IRBuilder::BuildReg1(Js::OpCode newOpcode, uint32 offset, Js::RegSlot R0)
14831483
{
14841484
case Js::OpCode::LdHeapArgsCached:
14851485
case Js::OpCode::LdLetHeapArgsCached:
1486-
this->m_func->SetHasArgumentObject();
14871486
if (!m_func->GetJnFunction()->HasScopeObject())
14881487
{
14891488
Js::Throw::FatalInternalError();
@@ -1724,7 +1723,6 @@ IRBuilder::BuildReg2(Js::OpCode newOpcode, uint32 offset, Js::RegSlot R0, Js::Re
17241723
dstSym->m_isSafeThis = true;
17251724
dstSym->m_isNotInt = true;
17261725
}
1727-
this->m_func->SetHasArgumentObject();
17281726
return;
17291727
}
17301728
case Js::OpCode::SetHomeObj:
@@ -6094,7 +6092,8 @@ IRBuilder::BuildProfiledCallI(Js::OpCode opcode, uint32 offset, Js::RegSlot retu
60946092
if(this->m_func->m_jitTimeData)
60956093
{
60966094
const Js::FunctionCodeGenJitTimeData *inlinerData = this->m_func->m_jitTimeData;
6097-
if(!this->IsLoopBody() && inlinerData->inlineesBv && (!inlinerData->inlineesBv->Test(profileId)
6095+
if(!(this->IsLoopBody() && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->m_func)) &&
6096+
inlinerData && inlinerData->inlineesBv && (!inlinerData->inlineesBv->Test(profileId)
60986097
#if DBG
60996098
|| (PHASE_STRESS(Js::BailOnNoProfilePhase, this->m_func->GetTopFunc()) &&
61006099
(CONFIG_FLAG(SkipFuncCountForBailOnNoProfile) < 0 ||

‎lib/Backend/Inline.cpp

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1907,7 +1907,7 @@ Inline::InlineBuiltInFunction(IR::Instr *callInstr, Js::FunctionInfo *funcInfo,
19071907

19081908
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
19091909
InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
1910-
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, profileId, builtInId);
1910+
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, profileId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
19111911
#endif
19121912

19131913
// From now on we are committed to inlining.
@@ -2255,7 +2255,7 @@ IR::Instr* Inline::InlineApply(IR::Instr *callInstr, Js::FunctionInfo *funcInfo,
22552255

22562256
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
22572257
InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
2258-
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, builtInId);
2258+
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
22592259
char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
22602260
#endif
22612261

@@ -2718,7 +2718,7 @@ Inline::InlineCall(IR::Instr *callInstr, Js::FunctionInfo *funcInfo, const Js::F
27182718

27192719
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
27202720
InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
2721-
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, builtInId);
2721+
nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
27222722
#endif
27232723

27242724
uint actualCount = 0;
@@ -3618,6 +3618,16 @@ Inline::InlineScriptFunction(IR::Instr *callInstr, const Js::FunctionCodeGenJitT
36183618
Js::FunctionBody *funcCaller = callInstr->m_func->GetJnFunction();
36193619
Js::FunctionBody *funcBody = inlineeData->GetFunctionBody();
36203620

3621+
// We don't do stack args optimization in jitted loop body (because of lack of information about the code before and after the loop)
3622+
// and we turn off stack arg optimization for the whole inline chain if we can't do it for one of the functionss.
3623+
// Inlining a function that uses arguments object could potentially hurt perf because we'll have to create arguments object on the
3624+
// heap for that function (versus otherwise the function will be jitted and have its arguments object creation optimized).
3625+
// TODO: Allow arguments object creation to be optimized on a function level instead of an all-or-nothing approach.
3626+
if (callInstr->m_func->IsLoopBody() && funcBody->GetUsesArgumentsObject())
3627+
{
3628+
return instrNext;
3629+
}
3630+
36213631
if (callInstr->GetSrc2() &&
36223632
callInstr->GetSrc2()->IsSymOpnd() &&
36233633
callInstr->GetSrc2()->AsSymOpnd()->m_sym->AsStackSym()->GetArgSlotNum() > Js::InlineeCallInfo::MaxInlineeArgoutCount)
@@ -5054,7 +5064,7 @@ Inline::HasArgumentsAccess(IR::Instr * instr, SymID argumentsSymId)
50545064
bool
50555065
Inline::GetInlineeHasArgumentObject(Func * inlinee)
50565066
{
5057-
if (!inlinee->GetHasArgumentObject())
5067+
if (!inlinee->GetJnFunction()->GetUsesArgumentsObject())
50585068
{
50595069
// If inlinee has no arguments access return false
50605070
return false;

‎lib/Backend/InliningDecider.cpp

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
#include "Backend.h"
66

77
InliningDecider::InliningDecider(Js::FunctionBody *const topFunc, bool isLoopBody, bool isInDebugMode, const ExecutionMode jitMode)
8-
: topFunc(topFunc), isLoopBody(isLoopBody), isInDebugMode(isInDebugMode), jitMode(jitMode), bytecodeInlinedCount(0), numberOfInlineesWithLoop (0), inliningHeuristics(topFunc)
8+
: topFunc(topFunc), isLoopBody(isLoopBody), isInDebugMode(isInDebugMode), jitMode(jitMode), bytecodeInlinedCount(0), numberOfInlineesWithLoop (0), inliningHeuristics(topFunc, isLoopBody)
99
{
1010
Assert(topFunc);
1111
}
@@ -190,7 +190,7 @@ Js::FunctionInfo *InliningDecider::Inline(Js::FunctionBody *const inliner, Js::F
190190
Js::FunctionProxy * proxy = functionInfo->GetFunctionProxy();
191191
if (proxy && proxy->IsFunctionBody())
192192
{
193-
if (isLoopBody && !PHASE_ON(Js::InlineInJitLoopBodyPhase, this->topFunc))
193+
if (isLoopBody && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->topFunc))
194194
{
195195
INLINE_TESTTRACE_VERBOSE(_u("INLINING: Skip Inline: Jit loop body: %s (%s)\n"), this->topFunc->GetDisplayName(),
196196
this->topFunc->GetDebugNumberSet(debugStringBuffer));
@@ -268,7 +268,7 @@ Js::FunctionInfo *InliningDecider::Inline(Js::FunctionBody *const inliner, Js::F
268268
}
269269

270270
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
271-
TraceInlining(inliner, inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inlinee->GetByteCodeCount(), this->topFunc, this->bytecodeInlinedCount, inlinee, callSiteId);
271+
TraceInlining(inliner, inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inlinee->GetByteCodeCount(), this->topFunc, this->bytecodeInlinedCount, inlinee, callSiteId, this->isLoopBody);
272272
#endif
273273

274274
this->bytecodeInlinedCount += inlinee->GetByteCodeCount();
@@ -608,32 +608,32 @@ bool InliningDecider::GetBuiltInInfo(
608608
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
609609
// static
610610
void InliningDecider::TraceInlining(Js::FunctionBody *const inliner, const char16* inlineeName, const char16* inlineeFunctionIdandNumberString, uint inlineeByteCodeCount,
611-
Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, uint builtIn)
611+
Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, bool inLoopBody, uint builtIn)
612612
{
613613
char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
614614
char16 debugStringBuffer2[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
615615
char16 debugStringBuffer3[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
616616
if (inlineeName == nullptr)
617617
{
618-
619618
int len = swprintf_s(debugStringBuffer3, MAX_FUNCTION_BODY_DEBUG_STRING_SIZE, _u("built In Id: %u"), builtIn);
620619
Assert(len > 14);
621620
inlineeName = debugStringBuffer3;
622621
}
623-
INLINE_TESTTRACE(_u("INLINING: Inlinee: %s (%s)\tSize: %d\tCaller: %s (%s)\tSize: %d\tInlineCount: %d\tRoot: %s (%s)\tSize: %d\tCallSiteId: %d\n"),
622+
INLINE_TESTTRACE(_u("INLINING %s: Inlinee: %s (%s)\tSize: %d\tCaller: %s (%s)\tSize: %d\tInlineCount: %d\tRoot: %s (%s)\tSize: %d\tCallSiteId: %d\n"),
623+
inLoopBody ? _u("IN LOOP BODY") : _u(""),
624624
inlineeName, inlineeFunctionIdandNumberString, inlineeByteCodeCount,
625-
inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(), inlinedByteCodeCount,
626-
topFunc->GetDisplayName(),
627-
topFunc->GetDebugNumberSet(debugStringBuffer2), topFunc->GetByteCodeCount(),
625+
inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(),
626+
inlinedByteCodeCount,
627+
topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2), topFunc->GetByteCodeCount(),
628628
callSiteId
629629
);
630630

631-
INLINE_TRACE(_u("INLINING:\n\tInlinee: size: %4d %s\n\tCaller: size: %4d %-25s (%s) InlineCount: %d\tRoot: size: %4d %s (%s) CallSiteId %d\n"),
632-
inlineeByteCodeCount, inlineeName,
633-
inliner->GetByteCodeCount(), inliner->GetDisplayName(),
634-
inliner->GetDebugNumberSet(debugStringBuffer), inlinedByteCodeCount,
635-
topFunc->GetByteCodeCount(),
636-
topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2),
631+
INLINE_TRACE(_u("INLINING %s: Inlinee: %s(%s)\tSize : %d\tCaller : %s(%s)\tSize : %d\tInlineCount : %d\tRoot : %s(%s)\tSize : %d\tCallSiteId : %d\n"),
632+
inLoopBody ? _u("IN LOOP BODY") : _u(""),
633+
inlineeName, inlineeFunctionIdandNumberString, inlineeByteCodeCount,
634+
inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(),
635+
inlinedByteCodeCount,
636+
topFunc->GetByteCodeCount(), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2),
637637
callSiteId
638638
);
639639

‎lib/Backend/InliningDecider.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ class InliningDecider
5151
bytecodeInlinedCount = 0;
5252
numberOfInlineesWithLoop = 0;
5353
}
54-
uint32 getNumberOfInlineesWithLoop() { return numberOfInlineesWithLoop; }
55-
void incrementNumberOfInlineesWithLoop() { numberOfInlineesWithLoop++; }
54+
uint32 GetNumberOfInlineesWithLoop() { return numberOfInlineesWithLoop; }
55+
void IncrementNumberOfInlineesWithLoop() { numberOfInlineesWithLoop++; }
5656

5757

5858
static bool GetBuiltInInfo(
@@ -63,7 +63,7 @@ class InliningDecider
6363

6464
#if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
6565
static void TraceInlining(Js::FunctionBody *const inliner, const char16* inlineeName, const char16* inlineeFunctionIdandNumberString, uint inlineeByteCodeCount,
66-
Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, uint builtIn = -1);
66+
Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, bool isLoopBody, uint builtIn = -1);
6767
#endif
6868

6969
PREVENT_COPY(InliningDecider)

0 commit comments

Comments
 (0)