Skip to content

Commit 28f8c96

Browse files
mkustermanncommit-bot@chromium.org
mkustermann
authored and
[email protected]
committed
[VM] Bare instructions - Part 5: Get rid of CODE_REG indirection in SwitchableCalls
If the --use-bare-instructions flag is enabled we will: * Make call sites load the target directly from the pool (instead of the code object) - this saves one instruction (and an indirect load) * Ensure the object pool will have direct entry addresses by: - Letting the clustered snapshot reader change any StubCode::UnlinkedCall() in the object pool by it's monomorphic entry - Change the code patcher to patch SwitchableCalls by writing the monomorphic entry into the pool (instead of the code object) Issue #33274 Change-Id: I4e41fc8e4461bde477cc559a6a4fccaaf3a350b5 Reviewed-on: https://dart-review.googlesource.com/c/86160 Commit-Queue: Martin Kustermann <[email protected]> Reviewed-by: Ryan Macnak <[email protected]> Reviewed-by: Vyacheslav Egorov <[email protected]>
1 parent f205292 commit 28f8c96

15 files changed

+444
-108
lines changed

‎runtime/vm/clustered_snapshot.cc

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5618,7 +5618,38 @@ RawApiError* FullSnapshotReader::ReadIsolateSnapshot() {
56185618
}
56195619
}
56205620

5621-
deserializer.ReadIsolateSnapshot(thread_->isolate()->object_store());
5621+
auto object_store = thread_->isolate()->object_store();
5622+
deserializer.ReadIsolateSnapshot(object_store);
5623+
5624+
#if defined(DART_PRECOMPILED_RUNTIME)
5625+
if (FLAG_use_bare_instructions) {
5626+
// By default, every switchable call site will put (ic_data, code) into the
5627+
// object pool. The [code] is initialized (at AOT compile-time) to be a
5628+
// [StubCode::UnlinkedCall].
5629+
//
5630+
// In --use-bare-instruction we reduce the extra indirection via the [code]
5631+
// object and store instead (ic_data, entrypoint) in the object pool.
5632+
//
5633+
// Since the actual [entrypoint] is only known at AOT runtime we switch all
5634+
// existing UnlinkedCall entries in the object pool to be it's entrypoint.
5635+
auto zone = thread_->zone();
5636+
const auto& pool = ObjectPool::Handle(
5637+
zone, ObjectPool::RawCast(object_store->global_object_pool()));
5638+
auto& entry = Object::Handle(zone);
5639+
auto& smi = Smi::Handle(zone);
5640+
for (intptr_t i = 0; i < pool.Length(); i++) {
5641+
if (pool.TypeAt(i) == ObjectPool::kTaggedObject) {
5642+
entry = pool.ObjectAt(i);
5643+
if (entry.raw() == StubCode::UnlinkedCall().raw()) {
5644+
smi = Smi::FromAlignedAddress(
5645+
StubCode::UnlinkedCall().MonomorphicEntryPoint());
5646+
pool.SetTypeAt(i, ObjectPool::kImmediate, ObjectPool::kPatchable);
5647+
pool.SetObjectAt(i, smi);
5648+
}
5649+
}
5650+
}
5651+
}
5652+
#endif // defined(DART_PRECOMPILED_RUNTIME)
56225653

56235654
return ApiError::null();
56245655
}

‎runtime/vm/code_patcher.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ WritableInstructionsScope::~WritableInstructionsScope() {
2828
}
2929
}
3030

31-
bool MatchesPattern(uword end, int16_t* pattern, intptr_t size) {
31+
bool MatchesPattern(uword end, const int16_t* pattern, intptr_t size) {
3232
// When breaking within generated code in GDB, it may overwrite individual
3333
// instructions with trap instructions, which can cause this test to fail.
3434
//

‎runtime/vm/code_patcher.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class CodePatcher : public AllStatic {
108108
// [0..255] values in [pattern] have to match, negative values are skipped.
109109
//
110110
// Example pattern: `[0x3d, 0x8b, -1, -1]`.
111-
bool MatchesPattern(uword end, int16_t* pattern, intptr_t size);
111+
bool MatchesPattern(uword end, const int16_t* pattern, intptr_t size);
112112

113113
class KBCPatcher : public AllStatic {
114114
public:

‎runtime/vm/code_patcher_arm.cc

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,23 +61,39 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
6161
const Object& data,
6262
const Code& target) {
6363
ASSERT(caller_code.ContainsInstructionAt(return_address));
64-
SwitchableCallPattern call(return_address, caller_code);
65-
call.SetData(data);
66-
call.SetTarget(target);
64+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
65+
BareSwitchableCallPattern call(return_address, caller_code);
66+
call.SetData(data);
67+
call.SetTarget(target);
68+
} else {
69+
SwitchableCallPattern call(return_address, caller_code);
70+
call.SetData(data);
71+
call.SetTarget(target);
72+
}
6773
}
6874

6975
RawCode* CodePatcher::GetSwitchableCallTargetAt(uword return_address,
7076
const Code& caller_code) {
7177
ASSERT(caller_code.ContainsInstructionAt(return_address));
72-
SwitchableCallPattern call(return_address, caller_code);
73-
return call.target();
78+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
79+
BareSwitchableCallPattern call(return_address, caller_code);
80+
return call.target();
81+
} else {
82+
SwitchableCallPattern call(return_address, caller_code);
83+
return call.target();
84+
}
7485
}
7586

7687
RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
7788
const Code& caller_code) {
7889
ASSERT(caller_code.ContainsInstructionAt(return_address));
79-
SwitchableCallPattern call(return_address, caller_code);
80-
return call.data();
90+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
91+
BareSwitchableCallPattern call(return_address, caller_code);
92+
return call.data();
93+
} else {
94+
SwitchableCallPattern call(return_address, caller_code);
95+
return call.data();
96+
}
8197
}
8298

8399
void CodePatcher::PatchNativeCallAt(uword return_address,

‎runtime/vm/code_patcher_arm64.cc

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -96,23 +96,39 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
9696
const Object& data,
9797
const Code& target) {
9898
ASSERT(caller_code.ContainsInstructionAt(return_address));
99-
SwitchableCallPattern call(return_address, caller_code);
100-
call.SetData(data);
101-
call.SetTarget(target);
99+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
100+
BareSwitchableCallPattern call(return_address, caller_code);
101+
call.SetData(data);
102+
call.SetTarget(target);
103+
} else {
104+
SwitchableCallPattern call(return_address, caller_code);
105+
call.SetData(data);
106+
call.SetTarget(target);
107+
}
102108
}
103109

104110
RawCode* CodePatcher::GetSwitchableCallTargetAt(uword return_address,
105111
const Code& caller_code) {
106112
ASSERT(caller_code.ContainsInstructionAt(return_address));
107-
SwitchableCallPattern call(return_address, caller_code);
108-
return call.target();
113+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
114+
BareSwitchableCallPattern call(return_address, caller_code);
115+
return call.target();
116+
} else {
117+
SwitchableCallPattern call(return_address, caller_code);
118+
return call.target();
119+
}
109120
}
110121

111122
RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
112123
const Code& caller_code) {
113124
ASSERT(caller_code.ContainsInstructionAt(return_address));
114-
SwitchableCallPattern call(return_address, caller_code);
115-
return call.data();
125+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
126+
BareSwitchableCallPattern call(return_address, caller_code);
127+
return call.data();
128+
} else {
129+
SwitchableCallPattern call(return_address, caller_code);
130+
return call.data();
131+
}
116132
}
117133

118134
void CodePatcher::PatchNativeCallAt(uword return_address,

‎runtime/vm/code_patcher_x64.cc

Lines changed: 137 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
#include "vm/dart_entry.h"
1313
#include "vm/instructions.h"
1414
#include "vm/object.h"
15+
#include "vm/object_store.h"
1516
#include "vm/raw_object.h"
17+
#include "vm/reverse_pc_lookup_cache.h"
1618

1719
namespace dart {
1820

@@ -209,12 +211,41 @@ class PoolPointerCall : public ValueObject {
209211
// load guarded cid load ICData load MegamorphicCache
210212
// load monomorphic target <-> load ICLookup stub -> load MMLookup stub
211213
// call target.entry call stub.entry call stub.entry
212-
class SwitchableCall : public ValueObject {
214+
class SwitchableCallBase : public ValueObject {
213215
public:
214-
SwitchableCall(uword return_address, const Code& code)
216+
explicit SwitchableCallBase(const Code& code)
215217
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
216218
target_index_(-1),
217-
data_index_(-1) {
219+
data_index_(-1) {}
220+
221+
intptr_t data_index() const { return data_index_; }
222+
intptr_t target_index() const { return target_index_; }
223+
224+
RawObject* data() const { return object_pool_.ObjectAt(data_index()); }
225+
226+
void SetData(const Object& data) const {
227+
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode());
228+
object_pool_.SetObjectAt(data_index(), data);
229+
// No need to flush the instruction cache, since the code is not modified.
230+
}
231+
232+
protected:
233+
ObjectPool& object_pool_;
234+
intptr_t target_index_;
235+
intptr_t data_index_;
236+
237+
private:
238+
DISALLOW_IMPLICIT_CONSTRUCTORS(SwitchableCallBase);
239+
};
240+
241+
// See [SwitchableCallBase] for a switchable calls in general.
242+
//
243+
// The target slot is always a [Code] object: Either the code of the
244+
// monomorphic function or a stub code.
245+
class SwitchableCall : public SwitchableCallBase {
246+
public:
247+
SwitchableCall(uword return_address, const Code& code)
248+
: SwitchableCallBase(code) {
218249
uword pc = return_address;
219250

220251
// callq RCX
@@ -277,33 +308,96 @@ class SwitchableCall : public ValueObject {
277308
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index_)).IsCode());
278309
}
279310

280-
intptr_t data_index() const { return data_index_; }
281-
intptr_t target_index() const { return target_index_; }
311+
void SetTarget(const Code& target) const {
312+
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode());
313+
object_pool_.SetObjectAt(target_index(), target);
314+
// No need to flush the instruction cache, since the code is not modified.
315+
}
282316

283-
RawObject* data() const { return object_pool_.ObjectAt(data_index()); }
284317
RawCode* target() const {
285318
return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_index()));
286319
}
320+
};
287321

288-
void SetData(const Object& data) const {
289-
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode());
290-
object_pool_.SetObjectAt(data_index(), data);
291-
// No need to flush the instruction cache, since the code is not modified.
322+
// See [SwitchableCallBase] for a switchable calls in general.
323+
//
324+
// The target slot is always a direct entrypoint address: Either the entry point
325+
// of the monomorphic function or a stub entry point.
326+
class BareSwitchableCall : public SwitchableCallBase {
327+
public:
328+
BareSwitchableCall(uword return_address, const Code& code)
329+
: SwitchableCallBase(code) {
330+
object_pool_ = ObjectPool::RawCast(
331+
Isolate::Current()->object_store()->global_object_pool());
332+
333+
uword pc = return_address;
334+
335+
// callq RCX
336+
static int16_t call_pattern[] = {
337+
0xff, 0xd1, //
338+
};
339+
if (MatchesPattern(pc, call_pattern, ARRAY_SIZE(call_pattern))) {
340+
pc -= ARRAY_SIZE(call_pattern);
341+
} else {
342+
FATAL1("Failed to decode at %" Px, pc);
343+
}
344+
345+
// movq RBX, [PP + offset]
346+
static int16_t load_data_disp8[] = {
347+
0x49, 0x8b, 0x5f, -1, //
348+
};
349+
static int16_t load_data_disp32[] = {
350+
0x49, 0x8b, 0x9f, -1, -1, -1, -1,
351+
};
352+
if (MatchesPattern(pc, load_data_disp8, ARRAY_SIZE(load_data_disp8))) {
353+
pc -= ARRAY_SIZE(load_data_disp8);
354+
data_index_ = IndexFromPPLoadDisp8(pc + 3);
355+
} else if (MatchesPattern(pc, load_data_disp32,
356+
ARRAY_SIZE(load_data_disp32))) {
357+
pc -= ARRAY_SIZE(load_data_disp32);
358+
data_index_ = IndexFromPPLoadDisp32(pc + 3);
359+
} else {
360+
FATAL1("Failed to decode at %" Px, pc);
361+
}
362+
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index_)).IsCode());
363+
364+
// movq RCX, [PP + offset]
365+
static int16_t load_code_disp8[] = {
366+
0x49, 0x8b, 0x4f, -1, //
367+
};
368+
static int16_t load_code_disp32[] = {
369+
0x49, 0x8b, 0x8f, -1, -1, -1, -1,
370+
};
371+
if (MatchesPattern(pc, load_code_disp8, ARRAY_SIZE(load_code_disp8))) {
372+
pc -= ARRAY_SIZE(load_code_disp8);
373+
target_index_ = IndexFromPPLoadDisp8(pc + 3);
374+
} else if (MatchesPattern(pc, load_code_disp32,
375+
ARRAY_SIZE(load_code_disp32))) {
376+
pc -= ARRAY_SIZE(load_code_disp32);
377+
target_index_ = IndexFromPPLoadDisp32(pc + 3);
378+
} else {
379+
FATAL1("Failed to decode at %" Px, pc);
380+
}
381+
ASSERT(object_pool_.TypeAt(target_index_) == ObjectPool::kImmediate);
292382
}
293383

294384
void SetTarget(const Code& target) const {
295-
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode());
296-
object_pool_.SetObjectAt(target_index(), target);
297-
// No need to flush the instruction cache, since the code is not modified.
385+
ASSERT(object_pool_.TypeAt(target_index()) == ObjectPool::kImmediate);
386+
object_pool_.SetRawValueAt(target_index(), target.MonomorphicEntryPoint());
298387
}
299388

300-
protected:
301-
const ObjectPool& object_pool_;
302-
intptr_t target_index_;
303-
intptr_t data_index_;
304-
305-
private:
306-
DISALLOW_IMPLICIT_CONSTRUCTORS(SwitchableCall);
389+
RawCode* target() const {
390+
const uword pc = object_pool_.RawValueAt(target_index());
391+
auto rct = Isolate::Current()->reverse_pc_lookup_cache();
392+
if (rct->Contains(pc)) {
393+
return rct->Lookup(pc);
394+
}
395+
rct = Dart::vm_isolate()->reverse_pc_lookup_cache();
396+
if (rct->Contains(pc)) {
397+
return rct->Lookup(pc);
398+
}
399+
UNREACHABLE();
400+
}
307401
};
308402

309403
RawCode* CodePatcher::GetStaticCallTargetAt(uword return_address,
@@ -360,23 +454,39 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
360454
const Object& data,
361455
const Code& target) {
362456
ASSERT(caller_code.ContainsInstructionAt(return_address));
363-
SwitchableCall call(return_address, caller_code);
364-
call.SetData(data);
365-
call.SetTarget(target);
457+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
458+
BareSwitchableCall call(return_address, caller_code);
459+
call.SetData(data);
460+
call.SetTarget(target);
461+
} else {
462+
SwitchableCall call(return_address, caller_code);
463+
call.SetData(data);
464+
call.SetTarget(target);
465+
}
366466
}
367467

368468
RawCode* CodePatcher::GetSwitchableCallTargetAt(uword return_address,
369469
const Code& caller_code) {
370470
ASSERT(caller_code.ContainsInstructionAt(return_address));
371-
SwitchableCall call(return_address, caller_code);
372-
return call.target();
471+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
472+
BareSwitchableCall call(return_address, caller_code);
473+
return call.target();
474+
} else {
475+
SwitchableCall call(return_address, caller_code);
476+
return call.target();
477+
}
373478
}
374479

375480
RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
376481
const Code& caller_code) {
377482
ASSERT(caller_code.ContainsInstructionAt(return_address));
378-
SwitchableCall call(return_address, caller_code);
379-
return call.data();
483+
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
484+
BareSwitchableCall call(return_address, caller_code);
485+
return call.data();
486+
} else {
487+
SwitchableCall call(return_address, caller_code);
488+
return call.data();
489+
}
380490
}
381491

382492
void CodePatcher::PatchNativeCallAt(uword return_address,

‎runtime/vm/compiler/assembler/assembler_x64.cc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,11 +1668,18 @@ void Assembler::MonomorphicCheckedEntry() {
16681668
movq(TMP, Immediate(kSmiCid));
16691669
jmp(&have_cid, kNearJump);
16701670

1671+
// Ensure the monomorphic entry is 2-byte aligned (so GC can see them if we
1672+
// store them in ICData / MegamorphicCache arrays)
1673+
nop(1);
1674+
16711675
Comment("MonomorphicCheckedEntry");
16721676
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
1677+
ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);
1678+
16731679
SmiUntag(RBX);
16741680
testq(RDI, Immediate(kSmiTagMask));
16751681
j(ZERO, &immediate, kNearJump);
1682+
nop(1);
16761683

16771684
LoadClassId(TMP, RDI);
16781685

0 commit comments

Comments
 (0)