Skip to content

Commit b11c932

Browse files
rmacnak-googleCommit Queue
authored and
Commit Queue
committed
[vm, gc] Work-steal the store buffer during scavenging.
TEST=ci Bug: #55713 Change-Id: Ie198a94c43f6ba8048695204eb716049bf8b23e2 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/369740 Reviewed-by: Alexander Aprelev <[email protected]> Commit-Queue: Ryan Macnak <[email protected]>
1 parent a75691f commit b11c932

File tree

1 file changed

+26
-15
lines changed

1 file changed

+26
-15
lines changed

runtime/vm/heap/scavenger.cc

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,9 @@ class ScavengerVisitorBase : public ObjectPointerVisitor,
143143
freelist_(freelist),
144144
bytes_promoted_(0),
145145
visiting_old_object_(nullptr),
146+
pending_(nullptr),
146147
promoted_list_(promotion_stack) {}
147-
~ScavengerVisitorBase() {}
148+
~ScavengerVisitorBase() { ASSERT(pending_ == nullptr); }
148149

149150
#ifdef DEBUG
150151
constexpr static const char* const kName = "Scavenger";
@@ -354,24 +355,31 @@ class ScavengerVisitorBase : public ObjectPointerVisitor,
354355
thread_ = nullptr;
355356
}
356357

357-
void Finalize() {
358+
void Finalize(StoreBuffer* store_buffer) {
358359
if (!scavenger_->abort_) {
359360
promoted_list_.Finalize();
360361
weak_array_list_.Finalize();
361362
weak_property_list_.Finalize();
362363
weak_reference_list_.Finalize();
363364
finalizer_entry_list_.Finalize();
365+
ASSERT(pending_ == nullptr);
364366
} else {
365367
promoted_list_.AbandonWork();
366368
weak_array_list_.AbandonWork();
367369
weak_property_list_.AbandonWork();
368370
weak_reference_list_.AbandonWork();
369371
finalizer_entry_list_.AbandonWork();
372+
if (pending_ != nullptr) {
373+
pending_->Reset();
374+
store_buffer->PushBlock(pending_, StoreBuffer::kIgnoreThreshold);
375+
pending_ = nullptr;
376+
}
370377
}
371378
}
372379

373380
Page* head() const { return head_; }
374381
Page* tail() const { return tail_; }
382+
void set_pending(StoreBufferBlock* pending) { pending_ = pending; }
375383

376384
static bool ForwardOrSetNullIfCollected(ObjectPtr parent,
377385
CompressedObjectPtr* ptr_address);
@@ -611,6 +619,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor,
611619
FreeList* freelist_;
612620
intptr_t bytes_promoted_;
613621
ObjectPtr visiting_old_object_;
622+
StoreBufferBlock* pending_;
614623
PromotionWorkList promoted_list_;
615624
LocalBlockWorkList<64, WeakArrayPtr> weak_array_list_;
616625
LocalBlockWorkList<64, WeakPropertyPtr> weak_property_list_;
@@ -1188,12 +1197,17 @@ template <bool parallel>
11881197
void Scavenger::IterateStoreBuffers(ScavengerVisitorBase<parallel>* visitor) {
11891198
TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "IterateStoreBuffers");
11901199

1191-
// Iterating through the store buffers.
1192-
// Grab the deduplication sets out of the isolate's consolidated store buffer.
11931200
StoreBuffer* store_buffer = heap_->isolate_group()->store_buffer();
1194-
StoreBufferBlock* pending = blocks_;
1195-
while (pending != nullptr) {
1196-
StoreBufferBlock* next = pending->next();
1201+
StoreBufferBlock* pending;
1202+
for (;;) {
1203+
{
1204+
MutexLocker ml(&space_lock_);
1205+
pending = blocks_;
1206+
if (pending == nullptr) break;
1207+
blocks_ = pending->next();
1208+
}
1209+
// Ensure the block is freed in case of scavenger abort.
1210+
visitor->set_pending(pending);
11971211
// Generated code appends to store buffers; tell MemorySanitizer.
11981212
MSAN_UNPOISON(pending, sizeof(*pending));
11991213
while (!pending->IsEmpty()) {
@@ -1207,9 +1221,8 @@ void Scavenger::IterateStoreBuffers(ScavengerVisitorBase<parallel>* visitor) {
12071221
pending->Reset();
12081222
// Return the emptied block for recycling (no need to check threshold).
12091223
store_buffer->PushBlock(pending, StoreBuffer::kIgnoreThreshold);
1210-
blocks_ = pending = next;
1224+
visitor->set_pending(nullptr);
12111225
}
1212-
// Done iterating through old objects remembered in the store buffers.
12131226
}
12141227

12151228
template <bool parallel>
@@ -1229,7 +1242,6 @@ void Scavenger::IterateObjectIdTable(ObjectPointerVisitor* visitor) {
12291242
enum RootSlices {
12301243
kIsolate = 0,
12311244
kObjectIdRing,
1232-
kStoreBuffer,
12331245
kNumRootSlices,
12341246
};
12351247

@@ -1248,14 +1260,12 @@ void Scavenger::IterateRoots(ScavengerVisitorBase<parallel>* visitor) {
12481260
case kObjectIdRing:
12491261
IterateObjectIdTable(visitor);
12501262
break;
1251-
case kStoreBuffer:
1252-
IterateStoreBuffers(visitor);
1253-
break;
12541263
default:
12551264
UNREACHABLE();
12561265
}
12571266
}
12581267

1268+
IterateStoreBuffers(visitor);
12591269
IterateRememberedCards(visitor);
12601270
}
12611271

@@ -1873,7 +1883,7 @@ intptr_t Scavenger::SerialScavenge(SemiSpace* from) {
18731883
visitor.ProcessRoots();
18741884
visitor.ProcessAll();
18751885
visitor.ProcessWeak();
1876-
visitor.Finalize();
1886+
visitor.Finalize(heap_->isolate_group()->store_buffer());
18771887
to_->AddList(visitor.head(), visitor.tail());
18781888
return visitor.bytes_promoted();
18791889
}
@@ -1907,9 +1917,10 @@ intptr_t Scavenger::ParallelScavenge(SemiSpace* from) {
19071917
}
19081918
}
19091919

1920+
StoreBuffer* store_buffer = heap_->isolate_group()->store_buffer();
19101921
for (intptr_t i = 0; i < num_tasks; i++) {
19111922
ParallelScavengerVisitor* visitor = visitors[i];
1912-
visitor->Finalize();
1923+
visitor->Finalize(store_buffer);
19131924
to_->AddList(visitor->head(), visitor->tail());
19141925
bytes_promoted += visitor->bytes_promoted();
19151926
delete visitor;

0 commit comments

Comments
 (0)