Skip to content

Commit 025a613

Browse files
azhoginazhogin
committed
Async drop codegen with async drop glue through templated coroutine
1 parent 536235f commit 025a613

File tree

98 files changed

+3385
-1928
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+3385
-1928
lines changed

Diff for: ‎compiler/rustc_borrowck/src/lib.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -696,7 +696,14 @@ impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
696696
TerminatorKind::SwitchInt { discr, targets: _ } => {
697697
self.consume_operand(loc, (discr, span), flow_state);
698698
}
699-
TerminatorKind::Drop { place, target: _, unwind: _, replace } => {
699+
TerminatorKind::Drop {
700+
place,
701+
target: _,
702+
unwind: _,
703+
replace,
704+
drop: _,
705+
async_fut: _,
706+
} => {
700707
debug!(
701708
"visit_terminator_drop \
702709
loc: {:?} term: {:?} place: {:?} span: {:?}",

Diff for: ‎compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
100100
TerminatorKind::SwitchInt { discr, targets: _ } => {
101101
self.consume_operand(location, discr);
102102
}
103-
TerminatorKind::Drop { place: drop_place, target: _, unwind: _, replace } => {
103+
TerminatorKind::Drop {
104+
place: drop_place,
105+
target: _,
106+
unwind: _,
107+
replace,
108+
drop: _,
109+
async_fut: _,
110+
} => {
104111
let write_kind =
105112
if *replace { WriteKind::Replace } else { WriteKind::StorageDeadOrDrop };
106113
self.access_place(

Diff for: ‎compiler/rustc_borrowck/src/type_check/mod.rs

+8-2
Original file line numberDiff line numberDiff line change
@@ -1690,8 +1690,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
16901690
}
16911691
}
16921692
TerminatorKind::Unreachable => {}
1693-
TerminatorKind::Drop { target, unwind, .. }
1694-
| TerminatorKind::Assert { target, unwind, .. } => {
1693+
TerminatorKind::Drop { target, unwind, drop, .. } => {
1694+
self.assert_iscleanup(body, block_data, target, is_cleanup);
1695+
self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
1696+
if let Some(drop) = drop {
1697+
self.assert_iscleanup(body, block_data, drop, is_cleanup);
1698+
}
1699+
}
1700+
TerminatorKind::Assert { target, unwind, .. } => {
16951701
self.assert_iscleanup(body, block_data, target, is_cleanup);
16961702
self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
16971703
}

Diff for: ‎compiler/rustc_codegen_cranelift/src/abi/mod.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
412412
Err(instance) => Some(instance),
413413
}
414414
}
415-
InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => {
415+
// We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
416+
// it is `func returning noop future`
417+
InstanceKind::DropGlue(_, None) => {
416418
// empty drop glue - a nop.
417419
let dest = target.expect("Non terminating drop_in_place_real???");
418420
let ret_block = fx.get_block(dest);
@@ -598,9 +600,8 @@ pub(crate) fn codegen_drop<'tcx>(
598600
let ty = drop_place.layout().ty;
599601
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
600602

601-
if let ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) =
602-
drop_instance.def
603-
{
603+
// AsyncDropGlueCtorShim can't be here
604+
if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def {
604605
// we don't actually need to drop anything
605606
} else {
606607
match ty.kind() {

Diff for: ‎compiler/rustc_codegen_cranelift/src/base.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
546546
| TerminatorKind::CoroutineDrop => {
547547
bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
548548
}
549-
TerminatorKind::Drop { place, target, unwind: _, replace: _ } => {
549+
TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => {
550+
assert!(
551+
async_fut.is_none() && drop.is_none(),
552+
"Async Drop must be expanded or reset to sync before codegen"
553+
);
550554
let drop_place = codegen_place(fx, *place);
551555
crate::abi::codegen_drop(fx, source_info, drop_place, *target);
552556
}

Diff for: ‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -683,8 +683,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>(
683683
_ => unreachable!(),
684684
};
685685

686-
let coroutine_layout =
687-
cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.kind_ty()).unwrap();
686+
let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap();
688687

689688
let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id);
690689
let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx);

Diff for: ‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,8 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>(
158158
DIFlags::FlagZero,
159159
),
160160
|cx, coroutine_type_di_node| {
161-
let coroutine_layout = cx
162-
.tcx
163-
.coroutine_layout(coroutine_def_id, coroutine_args.as_coroutine().kind_ty())
164-
.unwrap();
161+
let coroutine_layout =
162+
cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args).unwrap();
165163

166164
let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } =
167165
coroutine_type_and_layout.variants

Diff for: ‎compiler/rustc_codegen_ssa/src/back/symbol_export.rs

+42-2
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,16 @@ fn exported_symbols_provider_local(
383383
},
384384
));
385385
}
386+
MonoItem::Fn(Instance { def: InstanceKind::AsyncDropGlue(_, ty), args: _ }) => {
387+
symbols.push((
388+
ExportedSymbol::AsyncDropGlue(ty),
389+
SymbolExportInfo {
390+
level: SymbolExportLevel::Rust,
391+
kind: SymbolExportKind::Text,
392+
used: false,
393+
},
394+
));
395+
}
386396
_ => {
387397
// Any other symbols don't qualify for sharing
388398
}
@@ -406,6 +416,8 @@ fn upstream_monomorphizations_provider(
406416

407417
let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn();
408418
let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn();
419+
let async_drop_in_place_poll_fn_def_id = tcx.lang_items().async_drop_in_place_poll_fn();
420+
let future_drop_poll_fn_def_id = tcx.lang_items().future_drop_poll_fn();
409421

410422
for &cnum in cnums.iter() {
411423
for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() {
@@ -424,8 +436,20 @@ fn upstream_monomorphizations_provider(
424436
if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
425437
(async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
426438
} else {
427-
// `drop_in_place` in place does not exist, don't try
428-
// to use it.
439+
continue;
440+
}
441+
}
442+
ExportedSymbol::AsyncDropGlue(ty) => {
443+
if let Some(poll_fn_def_id) = async_drop_in_place_poll_fn_def_id {
444+
(poll_fn_def_id, tcx.mk_args(&[ty.into()]))
445+
} else {
446+
continue;
447+
}
448+
}
449+
ExportedSymbol::FutureDropPoll(ty) => {
450+
if let Some(poll_fn_def_id) = future_drop_poll_fn_def_id {
451+
(poll_fn_def_id, tcx.mk_args(&[ty.into()]))
452+
} else {
429453
continue;
430454
}
431455
}
@@ -577,6 +601,20 @@ pub fn symbol_name_for_instance_in_crate<'tcx>(
577601
instantiating_crate,
578602
)
579603
}
604+
ExportedSymbol::AsyncDropGlue(ty) => {
605+
rustc_symbol_mangling::symbol_name_for_instance_in_crate(
606+
tcx,
607+
Instance::resolve_async_drop_in_place_poll(tcx, ty),
608+
instantiating_crate,
609+
)
610+
}
611+
ExportedSymbol::FutureDropPoll(ty) => {
612+
rustc_symbol_mangling::symbol_name_for_instance_in_crate(
613+
tcx,
614+
Instance::resolve_future_drop_poll(tcx, ty),
615+
instantiating_crate,
616+
)
617+
}
580618
ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(),
581619
}
582620
}
@@ -628,6 +666,8 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
628666
// AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the
629667
// target's default symbol decoration scheme.
630668
ExportedSymbol::AsyncDropGlueCtorShim(..) => None,
669+
ExportedSymbol::AsyncDropGlue(..) => None,
670+
ExportedSymbol::FutureDropPoll(..) => None,
631671
// NoDefId always follow the target's default symbol decoration scheme.
632672
ExportedSymbol::NoDefId(..) => None,
633673
// ThreadLocalShim always follow the target's default symbol decoration scheme.

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/block.rs

+11-7
Original file line numberDiff line numberDiff line change
@@ -854,10 +854,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
854854

855855
let def = instance.map(|i| i.def);
856856

857-
if let Some(
858-
ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None),
859-
) = def
860-
{
857+
// We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
858+
// it is `func returning noop future`
859+
if let Some(ty::InstanceKind::DropGlue(_, None)) = def {
861860
// Empty drop glue; a no-op.
862861
let target = target.unwrap();
863862
return helper.funclet_br(self, bx, target, mergeable_succ);
@@ -1363,16 +1362,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
13631362
MergingSucc::False
13641363
}
13651364

1366-
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self
1367-
.codegen_drop_terminator(
1365+
mir::TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
1366+
assert!(
1367+
async_fut.is_none() && drop.is_none(),
1368+
"Async Drop must be expanded or reset to sync before codegen"
1369+
);
1370+
self.codegen_drop_terminator(
13681371
helper,
13691372
bx,
13701373
&terminator.source_info,
13711374
place,
13721375
target,
13731376
unwind,
13741377
mergeable_succ(),
1375-
),
1378+
)
1379+
}
13761380

13771381
mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
13781382
.codegen_assert_terminator(

Diff for: ‎compiler/rustc_const_eval/src/const_eval/machine.rs

+1
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
572572
RemainderByZero(op) => RemainderByZero(eval_to_int(op)?),
573573
ResumedAfterReturn(coroutine_kind) => ResumedAfterReturn(*coroutine_kind),
574574
ResumedAfterPanic(coroutine_kind) => ResumedAfterPanic(*coroutine_kind),
575+
ResumedAfterDrop(coroutine_kind) => ResumedAfterDrop(*coroutine_kind),
575576
MisalignedPointerDereference { ref required, ref found } => {
576577
MisalignedPointerDereference {
577578
required: eval_to_int(required)?,

Diff for: ‎compiler/rustc_const_eval/src/interpret/terminator.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
172172
}
173173
}
174174

175-
Drop { place, target, unwind, replace: _ } => {
175+
Drop { place, target, unwind, replace: _, drop: _, async_fut: _ } => {
176176
let place = self.eval_place(place)?;
177177
let instance = Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
178178
if let ty::InstanceKind::DropGlue(_, None) = instance.def {
@@ -587,6 +587,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
587587
| ty::InstanceKind::FnPtrAddrShim(..)
588588
| ty::InstanceKind::ThreadLocalShim(..)
589589
| ty::InstanceKind::AsyncDropGlueCtorShim(..)
590+
| ty::InstanceKind::AsyncDropGlue(..)
591+
| ty::InstanceKind::FutureDropPollShim(..)
590592
| ty::InstanceKind::Item(_) => {
591593
// We need MIR for this fn
592594
let Some((body, instance)) = M::find_mir_or_eval_fn(

Diff for: ‎compiler/rustc_feature/src/unstable.rs

+2
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ declare_features! (
357357
(unstable, associated_type_defaults, "1.2.0", Some(29661)),
358358
/// Allows `async || body` closures.
359359
(unstable, async_closure, "1.37.0", Some(62290)),
360+
/// Allows implementing `AsyncDrop`.
361+
(incomplete, async_drop, "CURRENT_RUSTC_VERSION", None),
360362
/// Allows `#[track_caller]` on async functions.
361363
(unstable, async_fn_track_caller, "1.73.0", Some(110011)),
362364
/// Allows `for await` loops.

Diff for: ‎compiler/rustc_hir/src/lang_items.rs

+9-13
Original file line numberDiff line numberDiff line change
@@ -167,19 +167,10 @@ language_item_table! {
167167

168168
Drop, sym::drop, drop_trait, Target::Trait, GenericRequirement::None;
169169
Destruct, sym::destruct, destruct_trait, Target::Trait, GenericRequirement::None;
170-
171-
AsyncDrop, sym::async_drop, async_drop_trait, Target::Trait, GenericRequirement::Exact(0);
172-
AsyncDestruct, sym::async_destruct, async_destruct_trait, Target::Trait, GenericRequirement::Exact(0);
170+
AsyncDrop, sym::async_drop, async_drop_trait, Target::Trait, GenericRequirement::None;
173171
AsyncDropInPlace, sym::async_drop_in_place, async_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1);
174-
SurfaceAsyncDropInPlace, sym::surface_async_drop_in_place, surface_async_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1);
175-
AsyncDropSurfaceDropInPlace, sym::async_drop_surface_drop_in_place, async_drop_surface_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1);
176-
AsyncDropSlice, sym::async_drop_slice, async_drop_slice_fn, Target::Fn, GenericRequirement::Exact(1);
177-
AsyncDropChain, sym::async_drop_chain, async_drop_chain_fn, Target::Fn, GenericRequirement::Exact(2);
178-
AsyncDropNoop, sym::async_drop_noop, async_drop_noop_fn, Target::Fn, GenericRequirement::Exact(0);
179-
AsyncDropDeferredDropInPlace, sym::async_drop_deferred_drop_in_place, async_drop_deferred_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1);
180-
AsyncDropFuse, sym::async_drop_fuse, async_drop_fuse_fn, Target::Fn, GenericRequirement::Exact(1);
181-
AsyncDropDefer, sym::async_drop_defer, async_drop_defer_fn, Target::Fn, GenericRequirement::Exact(1);
182-
AsyncDropEither, sym::async_drop_either, async_drop_either_fn, Target::Fn, GenericRequirement::Exact(3);
172+
AsyncDropInPlacePoll, sym::async_drop_in_place_poll, async_drop_in_place_poll_fn, Target::Closure, GenericRequirement::Exact(1);
173+
FutureDropPoll, sym::future_drop_poll, future_drop_poll_fn, Target::Fn, GenericRequirement::Exact(1);
183174

184175
CoerceUnsized, sym::coerce_unsized, coerce_unsized_trait, Target::Trait, GenericRequirement::Minimum(1);
185176
DispatchFromDyn, sym::dispatch_from_dyn, dispatch_from_dyn_trait, Target::Trait, GenericRequirement::Minimum(1);
@@ -294,6 +285,10 @@ language_item_table! {
294285
PanicAsyncFnResumedPanic, sym::panic_const_async_fn_resumed_panic, panic_const_async_fn_resumed_panic, Target::Fn, GenericRequirement::None;
295286
PanicAsyncGenFnResumedPanic, sym::panic_const_async_gen_fn_resumed_panic, panic_const_async_gen_fn_resumed_panic, Target::Fn, GenericRequirement::None;
296287
PanicGenFnNonePanic, sym::panic_const_gen_fn_none_panic, panic_const_gen_fn_none_panic, Target::Fn, GenericRequirement::None;
288+
PanicCoroutineResumedDrop, sym::panic_const_coroutine_resumed_drop, panic_const_coroutine_resumed_drop, Target::Fn, GenericRequirement::None;
289+
PanicAsyncFnResumedDrop, sym::panic_const_async_fn_resumed_drop, panic_const_async_fn_resumed_drop, Target::Fn, GenericRequirement::None;
290+
PanicAsyncGenFnResumedDrop, sym::panic_const_async_gen_fn_resumed_drop, panic_const_async_gen_fn_resumed_drop, Target::Fn, GenericRequirement::None;
291+
PanicGenFnNoneDrop, sym::panic_const_gen_fn_none_drop, panic_const_gen_fn_none_drop, Target::Fn, GenericRequirement::None;
297292
/// libstd panic entry point. Necessary for const eval to be able to catch it
298293
BeginPanic, sym::begin_panic, begin_panic_fn, Target::Fn, GenericRequirement::None;
299294

@@ -307,7 +302,7 @@ language_item_table! {
307302

308303
ExchangeMalloc, sym::exchange_malloc, exchange_malloc_fn, Target::Fn, GenericRequirement::None;
309304
DropInPlace, sym::drop_in_place, drop_in_place_fn, Target::Fn, GenericRequirement::Minimum(1);
310-
FallbackSurfaceDrop, sym::fallback_surface_drop, fallback_surface_drop_fn, Target::Fn, GenericRequirement::None;
305+
DropInPlaceFuture, sym::drop_in_place_future,drop_in_place_future_fn, Target::Fn, GenericRequirement::Exact(1);
311306
AllocLayout, sym::alloc_layout, alloc_layout, Target::Struct, GenericRequirement::None;
312307

313308
Start, sym::start, start_fn, Target::Fn, GenericRequirement::Exact(1);
@@ -361,6 +356,7 @@ language_item_table! {
361356
ResumeTy, sym::ResumeTy, resume_ty, Target::Struct, GenericRequirement::None;
362357
GetContext, sym::get_context, get_context_fn, Target::Fn, GenericRequirement::None;
363358

359+
Noop, sym::Noop, noop, Target::Struct, GenericRequirement::None;
364360
Context, sym::Context, context, Target::Struct, GenericRequirement::None;
365361
FuturePoll, sym::poll, future_poll_fn, Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
366362

Diff for: ‎compiler/rustc_hir_typeck/src/callee.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,9 @@ pub fn check_legal_trait_for_method_call(
3939
receiver: Option<Span>,
4040
expr_span: Span,
4141
trait_id: DefId,
42-
body_id: DefId,
42+
_body_id: DefId,
4343
) -> Result<(), ErrorGuaranteed> {
44-
if tcx.is_lang_item(trait_id, LangItem::Drop)
45-
&& tcx.lang_items().fallback_surface_drop_fn() != Some(body_id)
46-
{
44+
if tcx.is_lang_item(trait_id, LangItem::Drop) {
4745
let sugg = if let Some(receiver) = receiver.filter(|s| !s.is_empty()) {
4846
errors::ExplicitDestructorCallSugg::Snippet {
4947
lo: expr_span.shrink_to_lo(),

Diff for: ‎compiler/rustc_middle/messages.ftl

+6
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
middle_adjust_for_foreign_abi_error =
22
target architecture {$arch} does not support `extern {$abi}` ABI
33
4+
middle_assert_async_resume_after_drop = `async fn` resumed after async drop
5+
46
middle_assert_async_resume_after_panic = `async fn` resumed after panicking
57
68
middle_assert_async_resume_after_return = `async fn` resumed after completion
79
10+
middle_assert_coroutine_resume_after_drop = coroutine resumed after async drop
11+
812
middle_assert_coroutine_resume_after_panic = coroutine resumed after panicking
913
1014
middle_assert_coroutine_resume_after_return = coroutine resumed after completion
1115
1216
middle_assert_divide_by_zero =
1317
attempt to divide `{$val}` by zero
1418
19+
middle_assert_gen_resume_after_drop = `gen` fn or block cannot be further iterated on after it async dropped
20+
1521
middle_assert_gen_resume_after_panic = `gen` fn or block cannot be further iterated on after it panicked
1622
1723
middle_assert_misaligned_ptr_deref =

0 commit comments

Comments
 (0)