Skip to content

Commit 09fa1e9

Browse files
antonbashirantonbashir
committed
fibers
1 parent cf68474 commit 09fa1e9

21 files changed

+5741
-3500
lines changed

‎dart.S

-1
This file was deleted.

‎runtime/vm/compiler/runtime_api.h

+2
Original file line numberDiff line numberDiff line change
@@ -1265,6 +1265,8 @@ class Thread : public AllStatic {
12651265
static word no_scope_native_wrapper_entry_point_offset();
12661266
static word auto_scope_native_wrapper_entry_point_offset();
12671267

1268+
static word coroutine_offset();
1269+
12681270
#define THREAD_XMM_CONSTANT_LIST(V) \
12691271
V(float_not) \
12701272
V(float_negate) \

‎runtime/vm/compiler/runtime_offsets_extracted.h

+5,579-3,430
Large diffs are not rendered by default.

‎runtime/vm/compiler/runtime_offsets_list.h

+1
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@
350350
FIELD(Thread, random_offset) \
351351
FIELD(Thread, jump_to_frame_entry_point_offset) \
352352
FIELD(Thread, tsan_utils_offset) \
353+
FIELD(Thread, coroutine_offset) \
353354
FIELD(TsanUtils, setjmp_function_offset) \
354355
FIELD(TsanUtils, setjmp_buffer_offset) \
355356
FIELD(TsanUtils, exception_pc_offset) \

‎runtime/vm/compiler/stub_code_compiler.cc

+38-20
Original file line numberDiff line numberDiff line change
@@ -2213,19 +2213,24 @@ void StubCodeCompiler::GenerateInitSyncStarStub() {
22132213

22142214
void StubCodeCompiler::GenerateCoroutineInitializeStub() {
22152215
const Register kCoroutine = CoroutineInitializeStubABI::kCoroutineReg;
2216-
const Register kStackLimit = CoroutineInitializeStubABI::kStackLimitReg;
22172216

22182217
#if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64)
22192218
SPILLS_LR_TO_FRAME({});
22202219
#endif
2221-
__ LoadFieldFromOffset(kStackLimit, kCoroutine, target::Coroutine::stack_limit_offset());
2222-
__ StoreToOffset(kStackLimit, THR, Thread::stack_limit_offset());
2220+
__ EnterStubFrame();
2221+
__ PushObject(NullObject());
2222+
__ PushRegister(kCoroutine);
2223+
__ CallRuntime(kEnterCoroutineRuntimeEntry, 1);
2224+
__ Drop(2);
2225+
__ LeaveStubFrame();
22232226

22242227
__ PushRegister(FPREG);
22252228
__ PushRegister(PP);
22262229
__ PushRegister(CODE_REG);
22272230
__ PushRegister(FUNCTION_REG);
2231+
22282232
__ EnterFrame(0);
2233+
22292234
__ LoadFieldFromOffset(SPREG, kCoroutine, target::Coroutine::stack_base_offset());
22302235
__ PushRegister(FPREG);
22312236
__ LoadCompressedFieldFromOffset(FUNCTION_REG, kCoroutine, target::Coroutine::entry_offset());
@@ -2235,12 +2240,20 @@ void StubCodeCompiler::GenerateCoroutineInitializeStub() {
22352240
}
22362241
__ Call(FieldAddress(FUNCTION_REG, target::Function::entry_point_offset()));
22372242
__ PopRegister(FPREG);
2243+
22382244
__ LeaveFrame();
2245+
22392246
__ PopRegister(FUNCTION_REG);
22402247
__ PopRegister(CODE_REG);
22412248
__ PopRegister(PP);
22422249
__ PopRegister(FPREG);
22432250

2251+
__ EnterStubFrame();
2252+
__ PushObject(NullObject());
2253+
__ CallRuntime(kExitCoroutineRuntimeEntry, 0);
2254+
__ Drop(1);
2255+
__ LeaveStubFrame();
2256+
22442257
__ Ret();
22452258
}
22462259

@@ -2259,24 +2272,25 @@ void StubCodeCompiler::GenerateCoroutineTransferStub() {
22592272
__ PushRegister(FUNCTION_REG);
22602273
__ StoreFieldToOffset(SPREG, kFromCoroutine, target::Coroutine::stack_base_offset());
22612274

2262-
__ LoadFieldFromOffset(kToStackLimit, kToCoroutine, target::Coroutine::stack_limit_offset());
2263-
__ StoreToOffset(kToStackLimit, THR, Thread::stack_limit_offset());
2264-
22652275
__ LoadFieldFromOffset(SPREG, kToCoroutine, target::Coroutine::stack_base_offset());
22662276
__ PopRegister(FUNCTION_REG);
22672277
__ PopRegister(CODE_REG);
22682278
__ PopRegister(PP);
22692279
__ PopRegister(FPREG);
22702280

2281+
__ LoadFieldFromOffset(kToStackLimit, kToCoroutine, target::Coroutine::stack_limit_offset());
2282+
__ StoreToOffset(kToStackLimit, THR, Thread::stack_limit_offset());
2283+
__ StoreToOffset(kToCoroutine, THR, Thread::coroutine_offset());
2284+
22712285
__ StoreFieldToOffset(kFromCoroutine, kToCoroutine, target::Coroutine::caller_offset());
22722286

22732287
__ Ret();
22742288
}
22752289

22762290
void StubCodeCompiler::GenerateCoroutineForkStub() {
2277-
const Register kCaller = CoroutineForkStubABI::kCallerCoroutineReg;
2278-
const Register kForked = CoroutineForkStubABI::kForkedCoroutineReg;
2279-
const Register kForkedStackLimit = CoroutineForkStubABI::kForkedStackLimitReg;
2291+
const Register kCallerCoroutine = CoroutineForkStubABI::kCallerCoroutineReg;
2292+
const Register kForkedCoroutine = CoroutineForkStubABI::kForkedCoroutineReg;
2293+
const Register kStackLimit = CoroutineForkStubABI::kStackLimitReg;
22802294

22812295
#if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64)
22822296
SPILLS_LR_TO_FRAME({});
@@ -2286,27 +2300,31 @@ void StubCodeCompiler::GenerateCoroutineForkStub() {
22862300
__ PushRegister(PP);
22872301
__ PushRegister(CODE_REG);
22882302
__ PushRegister(FUNCTION_REG);
2289-
__ StoreFieldToOffset(SPREG, kCaller, target::Coroutine::stack_base_offset());
2303+
__ StoreFieldToOffset(SPREG, kCallerCoroutine, target::Coroutine::stack_base_offset());
22902304

2291-
__ LoadFieldFromOffset(kForkedStackLimit, kForked, target::Coroutine::stack_limit_offset());
2292-
__ StoreToOffset(kForkedStackLimit, THR, Thread::stack_limit_offset());
2293-
2294-
__ StoreFieldToOffset(kCaller, kForked, target::Coroutine::caller_offset());
2295-
__ LoadFieldFromOffset(SPREG, kForked, target::Coroutine::stack_base_offset());
2296-
__ PushRegister(kForked);
2297-
__ LoadCompressedFieldFromOffset(FUNCTION_REG, kForked, target::Coroutine::entry_offset());
2305+
__ StoreFieldToOffset(kCallerCoroutine, kForkedCoroutine, target::Coroutine::caller_offset());
2306+
__ LoadFieldFromOffset(SPREG, kForkedCoroutine, target::Coroutine::stack_base_offset());
2307+
__ PushRegister(kForkedCoroutine);
2308+
__ LoadCompressedFieldFromOffset(FUNCTION_REG, kForkedCoroutine, target::Coroutine::entry_offset());
22982309
if (!FLAG_precompiled_mode) {
22992310
__ LoadCompressedFieldFromOffset(CODE_REG, FUNCTION_REG, target::Function::code_offset());
23002311
__ LoadImmediate(ARGS_DESC_REG, 0);
23012312
}
2313+
2314+
__ LoadFieldFromOffset(kStackLimit, kForkedCoroutine, target::Coroutine::stack_limit_offset());
2315+
__ StoreToOffset(kStackLimit, THR, Thread::stack_limit_offset());
2316+
__ StoreToOffset(kForkedCoroutine, THR, Thread::coroutine_offset());
2317+
23022318
__ Call(FieldAddress(FUNCTION_REG, target::Function::entry_point_offset()));
2303-
__ PopRegister(kForked);
2304-
__ LoadFieldFromOffset(kCaller, kForked, target::Coroutine::caller_offset());
2305-
__ LoadFieldFromOffset(SPREG, kCaller, target::Coroutine::stack_base_offset());
2319+
2320+
__ PopRegister(kForkedCoroutine);
2321+
__ LoadFieldFromOffset(kCallerCoroutine, kForkedCoroutine, target::Coroutine::caller_offset());
2322+
__ LoadFieldFromOffset(SPREG, kCallerCoroutine, target::Coroutine::stack_base_offset());
23062323
__ PopRegister(FUNCTION_REG);
23072324
__ PopRegister(CODE_REG);
23082325
__ PopRegister(PP);
23092326
__ PopRegister(FPREG);
2327+
23102328
__ Ret();
23112329
}
23122330

‎runtime/vm/constants_x64.h

+1-2
Original file line numberDiff line numberDiff line change
@@ -390,14 +390,13 @@ struct SuspendStubABI {
390390

391391
struct CoroutineInitializeStubABI {
392392
static constexpr Register kCoroutineReg = RDI;
393-
static constexpr Register kStackLimitReg = RDX;
394393
static constexpr Register kEntryReg = RCX;
395394
};
396395

397396
struct CoroutineForkStubABI {
398397
static constexpr Register kCallerCoroutineReg = RDI;
399398
static constexpr Register kForkedCoroutineReg = RSI;
400-
static constexpr Register kForkedStackLimitReg = RDX;
399+
static constexpr Register kStackLimitReg = RDX;
401400
static constexpr Register kForkedEntryReg = RCX;
402401
};
403402

‎runtime/vm/isolate.h

+9-14
Original file line numberDiff line numberDiff line change
@@ -1004,6 +1004,9 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
10041004
T->field_table_values_ = field_table->table();
10051005
}
10061006

1007+
CoroutinePtr saved_coroutine() const { return saved_coroutine_; }
1008+
void save_coroutine(CoroutinePtr coroutine) { saved_coroutine_ = coroutine; }
1009+
10071010
IsolateObjectStore* isolate_object_store() const {
10081011
return isolate_object_store_.get();
10091012
}
@@ -1621,23 +1624,14 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
16211624
VMTagCounters vm_tag_counters_;
16221625

16231626
// We use 6 list entries for each pending service extension calls.
1624-
enum {
1625-
kPendingHandlerIndex = 0,
1626-
kPendingMethodNameIndex,
1627-
kPendingKeysIndex,
1628-
kPendingValuesIndex,
1629-
kPendingReplyPortIndex,
1630-
kPendingIdIndex,
1631-
kPendingEntrySize
1632-
};
1627+
enum {kPendingHandlerIndex = 0, kPendingMethodNameIndex, kPendingKeysIndex,
1628+
kPendingValuesIndex, kPendingReplyPortIndex, kPendingIdIndex,
1629+
kPendingEntrySize};
16331630
GrowableObjectArrayPtr pending_service_extension_calls_;
16341631

16351632
// We use 2 list entries for each registered extension handler.
1636-
enum {
1637-
kRegisteredNameIndex = 0,
1638-
kRegisteredHandlerIndex,
1639-
kRegisteredEntrySize
1640-
};
1633+
enum {kRegisteredNameIndex = 0, kRegisteredHandlerIndex,
1634+
kRegisteredEntrySize};
16411635
GrowableObjectArrayPtr registered_service_extension_handlers_;
16421636

16431637
// Used to wake the isolate when it is in the pause event loop.
@@ -1674,6 +1668,7 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
16741668
DeoptContext* deopt_context_ = nullptr;
16751669
FfiCallbackMetadata::Metadata* ffi_callback_list_head_ = nullptr;
16761670
intptr_t ffi_callback_keep_alive_counter_ = 0;
1671+
CoroutinePtr saved_coroutine_ = nullptr;
16771672

16781673
GrowableObjectArrayPtr tag_table_;
16791674

‎runtime/vm/object.cc

+6-6
Original file line numberDiff line numberDiff line change
@@ -26632,15 +26632,15 @@ CodePtr SuspendState::GetCodeObject() const {
2663226632
}
2663326633

2663426634
CoroutinePtr Coroutine::New(uintptr_t size, FunctionPtr entry) {
26635-
const auto& result = Coroutine::Handle(Object::Allocate<Coroutine>(Heap::kOld));
26635+
const auto& coroutine = Coroutine::Handle(Object::Allocate<Coroutine>(Heap::kOld));
2663626636
void** stack_base = (void**)((uintptr_t)mmap(0, size * sizeof(word), PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) + size);
2663726637
*(stack_base--) = 0;
2663826638
NoSafepointScope no_safepoint;
26639-
result.StoreNonPointer(&result.untag()->stack_base_, (uword)stack_base);
26640-
result.StoreNonPointer(&result.untag()->stack_limit_, (uword)stack_base - size);
26641-
result.StoreCompressedPointer(&result.untag()->entry_, entry);
26642-
result.StoreCompressedPointer(&result.untag()->caller_, result.ptr());
26643-
return result.ptr();
26639+
coroutine.StoreNonPointer(&coroutine.untag()->stack_base_, (uword)stack_base);
26640+
coroutine.StoreNonPointer(&coroutine.untag()->stack_limit_, (uword)stack_base - size);
26641+
coroutine.StoreCompressedPointer(&coroutine.untag()->entry_, entry);
26642+
coroutine.StoreCompressedPointer(&coroutine.untag()->caller_, coroutine.ptr());
26643+
return coroutine.ptr();
2664426644
}
2664526645

2664626646
const char* Coroutine::ToCString() const {

‎runtime/vm/object.h

+8-2
Original file line numberDiff line numberDiff line change
@@ -12705,17 +12705,23 @@ class Coroutine : public Instance {
1270512705
return RoundedAllocationSize(sizeof(UntaggedCoroutine));
1270612706
}
1270712707
static CoroutinePtr New(uintptr_t size, FunctionPtr entry);
12708+
12709+
CoroutinePtr caller() const { return untag()->caller(); }
1270812710
static uword caller_offset() { return OFFSET_OF(UntaggedCoroutine, caller_); }
12711+
12712+
FunctionPtr entry() const { return untag()->entry(); }
1270912713
static uword entry_offset() { return OFFSET_OF(UntaggedCoroutine, entry_); }
12714+
12715+
uword stack_base() const { return untag()->stack_base_; }
1271012716
static uword stack_base_offset() {
1271112717
return OFFSET_OF(UntaggedCoroutine, stack_base_);
1271212718
}
12719+
12720+
uword stack_limit() const { return untag()->stack_limit_; }
1271312721
static uword stack_limit_offset() {
1271412722
return OFFSET_OF(UntaggedCoroutine, stack_limit_);
1271512723
}
1271612724

12717-
CoroutinePtr caller() const { return untag()->caller(); }
12718-
1271912725
private:
1272012726
FINAL_HEAP_OBJECT_IMPLEMENTATION(Coroutine, Instance);
1272112727
friend class Class;

‎runtime/vm/os_thread.cc

-6
Original file line numberDiff line numberDiff line change
@@ -327,12 +327,6 @@ void OSThread::SetCurrentTLS(BaseThread* value) {
327327
}
328328
}
329329

330-
void OSThread::ChangeStackSize(uword new_base, uword new_size) {
331-
stack_base_ = new_base;
332-
stack_limit_ = new_base - new_size;
333-
stack_headroom_ = CalculateHeadroom(new_size);
334-
}
335-
336330
OSThreadIterator::OSThreadIterator() {
337331
ASSERT(OSThread::thread_list_lock_ != nullptr);
338332
// Lock the thread list while iterating.

‎runtime/vm/os_thread.h

-2
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,6 @@ class OSThread : public BaseThread {
125125

126126
Log* log() const { return log_; }
127127

128-
void ChangeStackSize(uword new_base, uword new_size);
129-
130128
uword stack_base() const { return stack_base_; }
131129
uword stack_limit() const { return stack_limit_; }
132130
uword overflow_stack_limit() const { return stack_limit_ + stack_headroom_; }

‎runtime/vm/profiler.cc

+7-2
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,13 @@ static bool GetAndValidateThreadStackBounds(OSThread* os_thread,
374374
#endif // defined(USING_SIMULATOR)
375375

376376
if (!use_simulator_stack_bounds) {
377-
*stack_lower = os_thread->stack_limit();
378-
*stack_upper = os_thread->stack_base();
377+
if (thread->coroutine() != nullptr) {
378+
*stack_lower = thread->coroutine()->untag()->stack_limit();
379+
*stack_upper = thread->coroutine()->untag()->stack_base();
380+
} else {
381+
*stack_lower = os_thread->stack_limit();
382+
*stack_upper = os_thread->stack_base();
383+
}
379384
}
380385

381386
if ((*stack_lower == 0) || (*stack_upper == 0)) {

‎runtime/vm/raw_object.h

+3
Original file line numberDiff line numberDiff line change
@@ -3783,6 +3783,9 @@ class UntaggedCoroutine : public UntaggedInstance {
37833783
CompressedObjectPtr* to_snapshot(Snapshot::Kind kind) { return to(); }
37843784
uword stack_base_;
37853785
uword stack_limit_;
3786+
public:
3787+
uword stack_base() const { return stack_base_; }
3788+
uword stack_limit() const { return stack_limit_; }
37863789
};
37873790

37883791
#undef WSR_COMPRESSED_POINTER_FIELD

‎runtime/vm/runtime_entry.cc

+11-8
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "vm/kernel_isolate.h"
2828
#include "vm/message.h"
2929
#include "vm/message_handler.h"
30+
#include "vm/native_entry.h"
3031
#include "vm/object_store.h"
3132
#include "vm/os_thread.h"
3233
#include "vm/parser.h"
@@ -3142,7 +3143,7 @@ DEFINE_RUNTIME_ENTRY(InterruptOrStackOverflow, 0) {
31423143
if (stack_pos == 0) {
31433144
// Use any reasonable value which would not be treated
31443145
// as stack overflow.
3145-
stack_pos = thread->saved_stack_limit();
3146+
stack_pos = thread->GetSavedStackLimit();
31463147
}
31473148
#else
31483149
uword stack_pos = OSThread::GetCurrentStackPointer();
@@ -3156,11 +3157,11 @@ DEFINE_RUNTIME_ENTRY(InterruptOrStackOverflow, 0) {
31563157
// process the stack overflow now and leave the interrupt for next
31573158
// time.
31583159
if (!thread->os_thread()->HasStackHeadroom() ||
3159-
IsCalleeFrameOf(thread->saved_stack_limit(), stack_pos)) {
3160+
IsCalleeFrameOf(thread->GetSavedStackLimit(), stack_pos)) {
31603161
if (FLAG_verbose_stack_overflow) {
31613162
OS::PrintErr("Stack overflow\n");
31623163
OS::PrintErr(" Native SP = %" Px ", stack limit = %" Px "\n", stack_pos,
3163-
thread->saved_stack_limit());
3164+
thread->GetSavedStackLimit());
31643165
OS::PrintErr("Call stack:\n");
31653166
OS::PrintErr("size | frame\n");
31663167
StackFrameIterator frames(ValidationPolicy::kDontValidateFrames, thread,
@@ -3872,11 +3873,13 @@ DEFINE_RUNTIME_ENTRY(FfiAsyncCallbackSend, 1) {
38723873
Message::New(target_port, handle, Message::kNormalPriority));
38733874
}
38743875

3875-
DEFINE_RUNTIME_ENTRY(ChangeThreadStackSize, 2) {
3876-
const auto& new_base = Smi::CheckedHandle(zone, arguments.NativeArgAt(0)).Value();
3877-
const auto& new_size = Smi::CheckedHandle(zone, arguments.NativeArgAt(1)).Value();
3878-
Thread::Current()->os_thread()->ChangeStackSize(new_base, new_size);
3879-
Thread::Current()->SetStackLimit(Thread::Current()->os_thread()->overflow_stack_limit());
3876+
DEFINE_RUNTIME_ENTRY(EnterCoroutine, 1) {
3877+
const Coroutine& coroutine = Coroutine::CheckedHandle(zone, arguments.ArgAt(0));
3878+
Thread::Current()->EnterCoroutine(coroutine.ptr());
3879+
}
3880+
3881+
DEFINE_RUNTIME_ENTRY(ExitCoroutine, 0) {
3882+
Thread::Current()->ExitCoroutine();
38803883
}
38813884

38823885
// Use expected function signatures to help MSVC compiler resolve overloading.

‎runtime/vm/runtime_entry_list.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ namespace dart {
7373
V(SwitchableCallMiss) \
7474
V(NotLoaded) \
7575
V(FfiAsyncCallbackSend) \
76-
V(ChangeThreadStackSize) \
76+
V(EnterCoroutine) \
77+
V(ExitCoroutine) \
7778

7879
// Note: Leaf runtime function have C linkage, so they cannot pass C++ struct
7980
// values like ObjectPtr.

‎runtime/vm/simulator_arm64.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -3913,7 +3913,7 @@ void Simulator::JumpToFrame(uword pc, uword sp, uword fp, Thread* thread) {
39133913
set_register(nullptr, SP, static_cast<int64_t>(sp));
39143914
set_register(nullptr, FP, static_cast<int64_t>(fp));
39153915
set_register(nullptr, THR, reinterpret_cast<int64_t>(thread));
3916-
set_register(nullptr, R31, thread->saved_stack_limit() - 4096);
3916+
set_register(nullptr, R31, thread->GetSavedStackLimit() - 4096);
39173917
#if defined(DART_TARGET_OS_FUCHSIA)
39183918
set_register(nullptr, R18, thread->saved_shadow_call_stack());
39193919
#endif

‎runtime/vm/stack_frame.cc

+4-3
Original file line numberDiff line numberDiff line change
@@ -607,9 +607,10 @@ void StackFrameIterator::FrameSetIterator::Unpoison() {
607607
#if !defined(USING_SIMULATOR)
608608
if (fp_ == 0) return;
609609
// Note that Thread::os_thread_ is cleared when the thread is descheduled.
610-
// ASSERT((thread_->os_thread() == nullptr) ||
611-
// ((thread_->os_thread()->stack_limit() < fp_) &&
612-
// (thread_->os_thread()->stack_base() > fp_)));
610+
ASSERT((thread_->coroutine() != nullptr) ||
611+
(thread_->os_thread() == nullptr) ||
612+
((thread_->os_thread()->stack_limit() < fp_) &&
613+
(thread_->os_thread()->stack_base() > fp_)));
613614
uword lower;
614615
if (sp_ == 0) {
615616
// Exit frame: guess sp.

0 commit comments

Comments
 (0)