From c9aacfea734f402b631e787c8b642af809428c2d Mon Sep 17 00:00:00 2001 From: Andrew Zhogin Date: Mon, 20 May 2024 14:13:51 +0700 Subject: [PATCH] Async drop glue integration in progress --- .../src/back/symbol_export.rs | 76 +++++++ compiler/rustc_codegen_ssa/src/mir/block.rs | 6 +- .../src/interpret/terminator.rs | 2 + compiler/rustc_hir/src/lang_items.rs | 3 +- compiler/rustc_hir_analysis/src/check/mod.rs | 5 + .../src/rmeta/decoder/cstore_impl.rs | 4 + .../src/middle/exported_symbols.rs | 8 + compiler/rustc_middle/src/mir/mod.rs | 17 ++ compiler/rustc_middle/src/mir/mono.rs | 7 +- compiler/rustc_middle/src/mir/pretty.rs | 11 + compiler/rustc_middle/src/mir/visit.rs | 6 +- compiler/rustc_middle/src/query/erase.rs | 2 + compiler/rustc_middle/src/query/mod.rs | 17 ++ compiler/rustc_middle/src/ty/adt.rs | 8 +- compiler/rustc_middle/src/ty/instance.rs | 45 +++- compiler/rustc_middle/src/ty/mod.rs | 12 +- compiler/rustc_middle/src/ty/util.rs | 79 +++++++ compiler/rustc_mir_build/src/build/scope.rs | 33 ++- compiler/rustc_mir_dataflow/Cargo.toml | 1 + .../rustc_mir_dataflow/src/elaborate_drops.rs | 215 ++++++++++++++---- compiler/rustc_mir_transform/src/coroutine.rs | 92 ++++++++ compiler/rustc_mir_transform/src/inline.rs | 30 ++- .../rustc_mir_transform/src/inline/cycle.rs | 7 +- compiler/rustc_mir_transform/src/shim.rs | 11 +- .../src/shim/async_destructor_ctor.rs | 184 +++++++++++++++ compiler/rustc_monomorphize/src/collector.rs | 6 +- .../rustc_monomorphize/src/partitioning.rs | 11 +- compiler/rustc_smir/src/rustc_smir/context.rs | 6 + .../rustc_smir/src/rustc_smir/convert/ty.rs | 6 +- compiler/rustc_span/src/symbol.rs | 1 + compiler/rustc_symbol_mangling/src/legacy.rs | 4 +- compiler/rustc_ty_utils/src/instance.rs | 21 ++ compiler/rustc_ty_utils/src/needs_drop.rs | 78 ++++++- compiler/stable_mir/src/compiler_interface.rs | 3 + compiler/stable_mir/src/mir/mono.rs | 5 +- library/core/src/future/async_drop.rs | 7 + .../async-await/async-drop/async-drop-open.rs | 128 +++++++++++ .../async-drop/async-drop-open.run.stdout | 11 + 38 files changed, 1079 insertions(+), 89 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs create mode 100644 tests/ui/async-await/async-drop/async-drop-open.rs create mode 100644 tests/ui/async-await/async-drop/async-drop-open.run.stdout diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 2dba04e0bb71b..4fcc7249cf1dc 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -363,6 +363,42 @@ fn exported_symbols_provider_local( }, )); } + MonoItem::Fn(Instance { + def: InstanceDef::AsyncDropGlueCtorShim(def_id, Some(ty)), + args, + }) => { + // A little sanity-check + debug_assert_eq!( + args.non_erasable_generics(tcx, def_id).skip(1).next(), + Some(GenericArgKind::Type(ty)) + ); + symbols.push(( + ExportedSymbol::AsyncDropGlueCtorShim(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } + MonoItem::Fn(Instance { + def: InstanceDef::AsyncDropGlue(def_id, Some(ty)), + args, + }) => { + // A little sanity-check + debug_assert_eq!( + args.non_erasable_generics(tcx, def_id).skip(1).next(), + Some(GenericArgKind::Type(ty)) + ); + symbols.push(( + ExportedSymbol::AsyncDropGlue(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } _ => { // Any other symbols don't qualify for sharing } @@ -385,6 +421,8 @@ fn upstream_monomorphizations_provider( let mut instances: DefIdMap> = Default::default(); let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn(); + let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn(); + let async_drop_in_place_poll_fn_def_id = tcx.lang_items().async_drop_in_place_poll_fn(); for &cnum in cnums.iter() { for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { @@ -399,6 +437,26 @@ fn upstream_monomorphizations_provider( continue; } } + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id { + ( + async_drop_in_place_fn_def_id, + tcx.mk_args(&[tcx.lifetimes.re_erased.into(), ty.into()]), + ) + } else { + continue; + } + } + ExportedSymbol::AsyncDropGlue(ty) => { + if let Some(poll_fn_def_id) = async_drop_in_place_poll_fn_def_id { + ( + poll_fn_def_id, + tcx.mk_args(&[tcx.lifetimes.re_erased.into(), ty.into()]), + ) + } else { + continue; + } + } ExportedSymbol::NonGeneric(..) | ExportedSymbol::ThreadLocalShim(..) | ExportedSymbol::NoDefId(..) => { @@ -534,6 +592,20 @@ pub fn symbol_name_for_instance_in_crate<'tcx>( Instance::resolve_drop_in_place(tcx, ty), instantiating_crate, ), + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::resolve_async_drop_in_place(tcx, ty), + instantiating_crate, + ) + } + ExportedSymbol::AsyncDropGlue(ty) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::resolve_async_drop_in_place_poll(tcx, ty), + instantiating_crate, + ) + } ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), } } @@ -581,6 +653,10 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>( // DropGlue always use the Rust calling convention and thus follow the target's default // symbol decoration scheme. ExportedSymbol::DropGlue(..) => None, + // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the + // target's default symbol decoration scheme. + ExportedSymbol::AsyncDropGlueCtorShim(..) => None, + ExportedSymbol::AsyncDropGlue(..) => None, // NoDefId always follow the target's default symbol decoration scheme. ExportedSymbol::NoDefId(..) => None, // ThreadLocalShim always follow the target's default symbol decoration scheme. diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 47402756c4267..ddd1e4d3e3c7c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -773,7 +773,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let def = instance.map(|i| i.def); - if let Some(ty::InstanceDef::DropGlue(_, None)) = def { + if let Some( + ty::InstanceDef::DropGlue(_, None) + | ty::InstanceDef::AsyncDropGlueCtorShim(_, None) + | ty::InstanceDef::AsyncDropGlue(_, None), + ) = def { // Empty drop glue; a no-op. let target = target.unwrap(); return helper.funclet_br(self, bx, target, mergeable_succ); diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 19c47f050228f..780fe37bbc8da 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -547,6 +547,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | ty::InstanceDef::CloneShim(..) | ty::InstanceDef::FnPtrAddrShim(..) | ty::InstanceDef::ThreadLocalShim(..) + | ty::InstanceDef::AsyncDropGlueCtorShim(..) + | ty::InstanceDef::AsyncDropGlue(..) | ty::InstanceDef::Item(_) => { // We need MIR for this fn let Some((body, instance)) = M::find_mir_or_eval_fn( diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 687d46d05b82c..2b7250780eeac 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -165,6 +165,7 @@ language_item_table! { Destruct, sym::destruct, destruct_trait, Target::Trait, GenericRequirement::None; AsyncDrop, sym::async_drop, async_drop_trait, Target::Trait, GenericRequirement::None; AsyncDropInPlace, sym::async_drop_in_place, async_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1); + AsyncDropInPlacePoll, sym::async_drop_in_place_poll, async_drop_in_place_poll_fn, Target::Closure, GenericRequirement::Exact(1); FutureDropPoll, sym::future_drop_poll, future_drop_poll_fn, Target::Fn, GenericRequirement::Exact(1); CoerceUnsized, sym::coerce_unsized, coerce_unsized_trait, Target::Trait, GenericRequirement::Minimum(1); @@ -263,7 +264,7 @@ language_item_table! { ExchangeMalloc, sym::exchange_malloc, exchange_malloc_fn, Target::Fn, GenericRequirement::None; DropInPlace, sym::drop_in_place, drop_in_place_fn, Target::Fn, GenericRequirement::Minimum(1); - DropInPlaceFuture, sym::drop_in_place_future,drop_in_place_future_fn, Target::Fn, GenericRequirement::Minimum(1); + DropInPlaceFuture, sym::drop_in_place_future,drop_in_place_future_fn, Target::Fn, GenericRequirement::Exact(1); AllocLayout, sym::alloc_layout, alloc_layout, Target::Struct, GenericRequirement::None; Start, sym::start, start_fn, Target::Fn, GenericRequirement::Exact(1); diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs index 28c8f846c23a9..6621183d9dcfc 100644 --- a/compiler/rustc_hir_analysis/src/check/mod.rs +++ b/compiler/rustc_hir_analysis/src/check/mod.rs @@ -110,6 +110,7 @@ pub fn provide(providers: &mut Providers) { wfcheck::provide(providers); *providers = Providers { adt_destructor, + adt_async_destructor, region_scope_tree, collect_return_position_impl_trait_in_trait_tys, compare_impl_const: compare_impl_item::compare_impl_const_raw, @@ -122,6 +123,10 @@ fn adt_destructor(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option tcx.calculate_dtor(def_id.to_def_id(), dropck::check_drop_impl) } +fn adt_async_destructor(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option { + tcx.calculate_async_dtor(def_id.to_def_id(), dropck::check_drop_impl) +} + /// Given a `DefId` for an opaque type in return position, find its parent item's return /// expressions. fn get_owner_return_paths( diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 0b352a02b64c7..7cca5af44f5f1 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -286,6 +286,10 @@ provide! { tcx, def_id, other, cdata, let _ = cdata; tcx.calculate_dtor(def_id, |_,_| Ok(())) } + adt_async_destructor => { + let _ = cdata; + tcx.calculate_async_dtor(def_id, |_,_| Ok(())) + } associated_item_def_ids => { tcx.arena.alloc_from_iter(cdata.get_associated_item_or_field_def_ids(def_id.index)) } diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index e30b6b203d73c..28b128c35aa6a 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -43,6 +43,8 @@ pub enum ExportedSymbol<'tcx> { NonGeneric(DefId), Generic(DefId, GenericArgsRef<'tcx>), DropGlue(Ty<'tcx>), + AsyncDropGlueCtorShim(Ty<'tcx>), + AsyncDropGlue(Ty<'tcx>), ThreadLocalShim(DefId), NoDefId(ty::SymbolName<'tcx>), } @@ -59,6 +61,12 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::DropGlue(ty) => { tcx.symbol_name(ty::Instance::resolve_drop_in_place(tcx, ty)) } + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + tcx.symbol_name(ty::Instance::resolve_async_drop_in_place(tcx, ty)) + } + ExportedSymbol::AsyncDropGlue(ty) => { + tcx.symbol_name(ty::Instance::resolve_async_drop_in_place_poll(tcx, ty)) + } ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance { def: ty::InstanceDef::ThreadLocalShim(def_id), args: ty::GenericArgs::empty(), diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index f6815ab1c9552..59e65fd31c2a2 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -258,6 +258,9 @@ pub struct CoroutineInfo<'tcx> { /// Coroutine async drop glue. pub coroutine_drop_async: Option>, + /// When coroutine has sync drop, this is async proxy calling `coroutine_drop` sync impl. + pub coroutine_drop_proxy_async: Option>, + /// The layout of a coroutine. Produced by the state transformation. pub coroutine_layout: Option>, @@ -279,6 +282,7 @@ impl<'tcx> CoroutineInfo<'tcx> { resume_ty: Some(resume_ty), coroutine_drop: None, coroutine_drop_async: None, + coroutine_drop_proxy_async: None, coroutine_layout: None, } } @@ -587,6 +591,19 @@ impl<'tcx> Body<'tcx> { self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop_async.as_ref()) } + #[inline] + pub fn coroutine_requires_async_drop(&self) -> bool { + self.coroutine_drop_async().is_some() + } + + #[inline] + pub fn future_drop_poll(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref().and_then( + |coroutine| + coroutine.coroutine_drop_async.as_ref().or( + coroutine.coroutine_drop_proxy_async.as_ref())) + } + #[inline] pub fn coroutine_kind(&self) -> Option { self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind) diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index a8f20e206918e..e1539cce76855 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -64,7 +64,10 @@ impl<'tcx> MonoItem<'tcx> { match instance.def { // "Normal" functions size estimate: the number of // statements, plus one for the terminator. - InstanceDef::Item(..) | InstanceDef::DropGlue(..) => { + InstanceDef::Item(..) + | InstanceDef::DropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) + | InstanceDef::AsyncDropGlue(..) => { let mir = tcx.instance_mir(instance.def); mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum() } @@ -404,9 +407,11 @@ impl<'tcx> CodegenUnit<'tcx> { | InstanceDef::ClosureOnceShim { .. } | InstanceDef::FutureDropPollShim(..) | InstanceDef::DropGlue(..) + | InstanceDef::AsyncDropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::ThreadLocalShim(..) | InstanceDef::FnPtrAddrShim(..) => None, + | InstanceDef::AsyncDropGlueCtorShim(..) => None, } } MonoItem::Static(def_id) => def_id.as_local().map(Idx::index), diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index d0ed3b36373a3..70df00cc63764 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -185,6 +185,17 @@ fn dump_path<'tcx>( })); s } + ty::InstanceDef::AsyncDropGlueCtorShim(_, Some(ty)) => { + // Unfortunately, pretty-printed typed are not very filename-friendly. + // We dome some filtering. + let mut s = ".".to_owned(); + s.extend(ty.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s + } _ => String::new(), }; diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 30767e8e891f7..7d570abd7c2d9 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -345,13 +345,17 @@ macro_rules! make_mir_visitor { ty::InstanceDef::Virtual(_def_id, _) | ty::InstanceDef::ThreadLocalShim(_def_id) | ty::InstanceDef::ClosureOnceShim { call_once: _def_id, track_caller: _ } | + ty::InstanceDef::AsyncDropGlueCtorShim(_def_id, None) | + ty::InstanceDef::AsyncDropGlue(_def_id, None) | ty::InstanceDef::DropGlue(_def_id, None) => {} ty::InstanceDef::FnPtrShim(_def_id, ty) | ty::InstanceDef::FutureDropPollShim(_def_id, ty) | ty::InstanceDef::DropGlue(_def_id, Some(ty)) | ty::InstanceDef::CloneShim(_def_id, ty) | - ty::InstanceDef::FnPtrAddrShim(_def_id, ty) => { + ty::InstanceDef::FnPtrAddrShim(_def_id, ty) | + ty::InstanceDef::AsyncDropGlue(_def_id, Some(ty)) | + ty::InstanceDef::AsyncDropGlueCtorShim(_def_id, Some(ty)) => { // FIXME(eddyb) use a better `TyContext` here. self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); } diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index 1b01df6a18714..5aee368f68eae 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -232,6 +232,7 @@ trivial! { Option, Option, Option, + Option, Option, Option, Option, @@ -286,6 +287,7 @@ trivial! { rustc_middle::ty::AssocItem, rustc_middle::ty::AssocItemContainer, rustc_middle::ty::Asyncness, + rustc_middle::ty::AsyncDestructor, rustc_middle::ty::BoundVariableKind, rustc_middle::ty::DeducedParamAttrs, rustc_middle::ty::Destructor, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 1d454a44492cb..1b6e0fe36f912 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -711,6 +711,11 @@ rustc_queries! { cache_on_disk_if { key.is_local() } separate_provide_extern } + query adt_async_destructor(key: DefId) -> Option { + desc { |tcx| "computing `AsyncDrop` impl for `{}`", tcx.def_path_str(key) } + cache_on_disk_if { key.is_local() } + separate_provide_extern + } query adt_sized_constraint(key: DefId) -> ty::EarlyBinder<&'tcx ty::List>> { desc { |tcx| "computing `Sized` constraints for `{}`", tcx.def_path_str(key) } @@ -1355,6 +1360,10 @@ rustc_queries! { query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { desc { "computing whether `{}` needs drop", env.value } } + /// Query backing `Ty::needs_async_drop`. + query needs_async_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + desc { "computing whether `{}` needs async drop", env.value } + } /// Query backing `Ty::has_significant_drop_raw`. query has_significant_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { desc { "computing whether `{}` has a significant drop", env.value } @@ -1379,6 +1388,14 @@ rustc_queries! { cache_on_disk_if { true } } + /// A list of types where the ADT requires async drop if and only if any of + /// those types require async drop. If the ADT is known to always need async drop + /// then `Err(AlwaysRequiresDrop)` is returned. + query adt_async_drop_tys(def_id: DefId) -> Result<&'tcx ty::List>, AlwaysRequiresDrop> { + desc { |tcx| "computing when `{}` needs async drop", tcx.def_path_str(def_id) } + cache_on_disk_if { true } + } + /// A list of types where the ADT requires drop if and only if any of those types /// has significant drop. A type marked with the attribute `rustc_insignificant_dtor` /// is considered to not be significant. A drop is significant if it is implemented diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 685c3e87dac1f..9235ec1572fbe 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -22,7 +22,9 @@ use std::hash::{Hash, Hasher}; use std::ops::Range; use std::str; -use super::{Destructor, FieldDef, GenericPredicates, Ty, TyCtxt, VariantDef, VariantDiscr}; +use super::{ + AsyncDestructor, Destructor, FieldDef, GenericPredicates, Ty, TyCtxt, VariantDef, VariantDiscr +}; #[derive(Clone, Copy, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable)] pub struct AdtFlags(u16); @@ -564,6 +566,10 @@ impl<'tcx> AdtDef<'tcx> { tcx.adt_destructor(self.did()) } + pub fn async_destructor(self, tcx: TyCtxt<'tcx>) -> Option { + tcx.adt_async_destructor(self.did()) + } + /// Returns a list of types such that `Self: Sized` if and only if that /// type is `Sized`, or `ty::Error` if this type has a recursive layout. pub fn sized_constraint(self, tcx: TyCtxt<'tcx>) -> ty::EarlyBinder<&'tcx ty::List>> { diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index a6a687f764bdf..ea3bb12d798fe 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -116,6 +116,15 @@ pub enum InstanceDef<'tcx> { /// /// The `DefId` is for `FnPtr::addr`, the `Ty` is the type `T`. FnPtrAddrShim(DefId, Ty<'tcx>), + + /// `core::future::async_drop::async_drop_in_place::<'_, T>`. + /// + /// The `DefId` is for `core::future::async_drop::async_drop_in_place`, the `Ty` + /// is the type `T`. + AsyncDropGlueCtorShim(DefId, Option>), + + /// async_drop_in_place poll function implementation + AsyncDropGlue(DefId, Option>), } impl<'tcx> Instance<'tcx> { @@ -153,7 +162,11 @@ impl<'tcx> Instance<'tcx> { InstanceDef::Item(def) => tcx .upstream_monomorphizations_for(def) .and_then(|monos| monos.get(&self.args).cloned()), - InstanceDef::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args), + InstanceDef::DropGlue(_, Some(_)) + | InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) + | InstanceDef::AsyncDropGlue(_, Some(_)) => { + tcx.upstream_drop_glue_for(self.args) + } _ => None, } } @@ -174,7 +187,9 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::FutureDropPollShim(def_id, _) | InstanceDef::DropGlue(def_id, _) | InstanceDef::CloneShim(def_id, _) - | InstanceDef::FnPtrAddrShim(def_id, _) => def_id, + | InstanceDef::FnPtrAddrShim(def_id, _) + | InstanceDef::AsyncDropGlue(def_id, _) + | InstanceDef::AsyncDropGlueCtorShim(def_id, _) => def_id, } } @@ -183,6 +198,8 @@ impl<'tcx> InstanceDef<'tcx> { match self { ty::InstanceDef::Item(def) => Some(def), ty::InstanceDef::DropGlue(def_id, Some(_)) + | InstanceDef::AsyncDropGlueCtorShim(def_id, Some(_)) + | InstanceDef::AsyncDropGlue(def_id, Some(_)) | InstanceDef::ThreadLocalShim(def_id) | ty::InstanceDef::FutureDropPollShim(def_id, ..) => Some(def_id), InstanceDef::VTableShim(..) @@ -192,6 +209,8 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::Intrinsic(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::DropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) + | InstanceDef::AsyncDropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::FnPtrAddrShim(..) => None, } @@ -216,6 +235,8 @@ impl<'tcx> InstanceDef<'tcx> { let def_id = match *self { ty::InstanceDef::Item(def) => def, ty::InstanceDef::DropGlue(_, Some(_)) => return false, + ty::InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) => return false, + ty::InstanceDef::AsyncDropGlue(_, Some(_)) => return false, ty::InstanceDef::ThreadLocalShim(_) => return false, _ => return true, }; @@ -286,8 +307,12 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::FnPtrShim(..) | InstanceDef::FutureDropPollShim(..) | InstanceDef::DropGlue(_, Some(_)) => false, + | InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) => false, + | InstanceDef::AsyncDropGlue(_, Some(_)) => false, InstanceDef::ClosureOnceShim { .. } | InstanceDef::DropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) + | InstanceDef::AsyncDropGlue(..) | InstanceDef::Item(_) | InstanceDef::Intrinsic(..) | InstanceDef::ReifyShim(..) @@ -327,8 +352,12 @@ fn fmt_instance( InstanceDef::FutureDropPollShim(_, ty) => write!(f, " - shim({ty})"), InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"), InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), + InstanceDef::AsyncDropGlue(_, None) => write!(f, " - shim(None)"), + InstanceDef::AsyncDropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), InstanceDef::CloneShim(_, ty) => write!(f, " - shim({ty})"), InstanceDef::FnPtrAddrShim(_, ty) => write!(f, " - shim({ty})"), + InstanceDef::AsyncDropGlueCtorShim(_, None) => write!(f, " - shim(None)"), + InstanceDef::AsyncDropGlueCtorShim(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), } } @@ -551,6 +580,18 @@ impl<'tcx> Instance<'tcx> { Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args) } + pub fn resolve_async_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> { + let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, None); + let args = tcx.mk_args(&[ty.into()]); + Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args) + } + + pub fn resolve_async_drop_in_place_poll(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> { + let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlacePoll, None); + let args = tcx.mk_args(&[ty.into()]); + Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args) + } + #[instrument(level = "debug", skip(tcx), ret)] pub fn fn_once_adapter_instance( tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 36490a49c4b23..fcb06895a90a6 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1793,6 +1793,14 @@ pub struct Destructor { pub constness: hir::Constness, } +#[derive(Copy, Clone, Debug, HashStable, Encodable, Decodable)] +pub struct AsyncDestructor { + /// The `DefId` of the async destructor future constructor + pub ctor: DefId, + /// The `DefId` of the async destructor future type + pub future: DefId, +} + #[derive(Clone, Copy, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)] pub struct VariantFlags(u8); bitflags! { @@ -2357,7 +2365,9 @@ impl<'tcx> TyCtxt<'tcx> { | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) | ty::InstanceDef::ThreadLocalShim(..) - | ty::InstanceDef::FnPtrAddrShim(..) => self.mir_shims(instance), + | ty::InstanceDef::FnPtrAddrShim(..) + | ty::InstanceDef::AsyncDropGlue(..) + | ty::InstanceDef::AsyncDropGlueCtorShim(..) => self.mir_shims(instance), } } diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index a6e50b9823305..7f8db1d04a8e5 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -380,6 +380,45 @@ impl<'tcx> TyCtxt<'tcx> { Some(ty::Destructor { did, constness }) } + /// Calculate the async destructor of a given type. + pub fn calculate_async_dtor( + self, + adt_did: DefId, + validate: impl Fn(Self, DefId) -> Result<(), ErrorGuaranteed>, + ) -> Option { + let async_drop_trait = self.lang_items().async_drop_trait()?; + self.ensure().coherent_trait(async_drop_trait).ok()?; + + let ty = self.type_of(adt_did).instantiate_identity(); + let mut dtor_candidate = None; + self.for_each_relevant_impl(async_drop_trait, ty, |impl_did| { + if validate(self, impl_did).is_err() { + // Already `ErrorGuaranteed`, no need to delay a span bug here. + return; + } + + let [future, ctor] = self.associated_item_def_ids(impl_did) else { + self.dcx().span_delayed_bug( + self.def_span(impl_did), + "AsyncDrop impl without async_drop function or Dropper type", + ); + return; + }; + + if let Some((_, _, old_impl_did)) = dtor_candidate { + self.dcx() + .struct_span_err(self.def_span(impl_did), "multiple async drop impls found") + .with_span_note(self.def_span(old_impl_did), "other impl here") + .delay_as_bug(); + } + + dtor_candidate = Some((*future, *ctor, impl_did)); + }); + + let (future, ctor, _) = dtor_candidate?; + Some(ty::AsyncDestructor { future, ctor }) + } + /// Returns the set of types that are required to be alive in /// order to run the destructor of `def` (see RFCs 769 and /// 1238). @@ -1243,6 +1282,46 @@ impl<'tcx> Ty<'tcx> { } } + /// If `ty.needs_async_drop(...)` returns `true`, then `ty` is definitely + /// non-copy and *might* have a async destructor attached; if it returns + /// `false`, then `ty` definitely has no async destructor (i.e., no async + /// drop glue). + /// + /// (Note that this implies that if `ty` has an async destructor attached, + /// then `needs_async_drop` will definitely return `true` for `ty`.) + /// + /// When constructing `AsyncDestruct::Destructor` type, use + /// [`Ty::async_drop_glue_morphology`] instead. + // + // FIXME(zetanumbers): Note that this method is used to check eligible types + // in unions. + #[inline] + pub fn needs_async_drop(self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool { + // Avoid querying in simple cases. + match needs_drop_components(tcx, self) { + Err(AlwaysRequiresDrop) => true, + Ok(components) => { + let query_ty = match *components { + [] => return false, + // If we've got a single component, call the query with that + // to increase the chance that we hit the query cache. + [component_ty] => component_ty, + _ => self, + }; + + // This doesn't depend on regions, so try to minimize distinct + // query keys used. + // If normalization fails, we just use `query_ty`. + debug_assert!(!param_env.has_infer()); + let query_ty = tcx + .try_normalize_erasing_regions(param_env, query_ty) + .unwrap_or_else(|_| tcx.erase_regions(query_ty)); + + tcx.needs_async_drop_raw(param_env.and(query_ty)) + } + } + } + /// Checks if `ty` has a significant drop. /// /// Note that this method can return false even if `ty` has a destructor diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs index 9936831fc6cab..d74eb53a3fca2 100644 --- a/compiler/rustc_mir_build/src/build/scope.rs +++ b/compiler/rustc_mir_build/src/build/scope.rs @@ -94,6 +94,7 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_session::lint::Level; use rustc_span::source_map::Spanned; use rustc_span::{Span, DUMMY_SP}; +use rustc_target::abi::FIRST_VARIANT; #[derive(Debug)] pub struct Scopes<'tcx> { @@ -706,6 +707,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume); } + fn may_needs_async_drop( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + param_env: ty::ParamEnv<'tcx>, + local: Local, + ) -> bool { + let ty = local_decls[local].ty; + if let ty::Adt(def, args) = ty.kind() { + if def.is_struct() && Some(def.did()) != tcx.lang_items().manually_drop() { + return def.variant(FIRST_VARIANT).fields.iter().any(|f| { + let field_ty = tcx.normalize_erasing_regions(param_env, f.ty(tcx, args)); + field_ty.is_async_drop(tcx, param_env) + }); + } + } + false + } + fn is_async_drop_impl( tcx: TyCtxt<'tcx>, local_decls: &IndexVec>, @@ -718,7 +737,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { return true; } } - ty.is_async_drop(tcx, param_env) || ty.is_coroutine() + if ty.is_async_drop(tcx, param_env) || ty.is_coroutine() { + return true; + } + //Self::may_needs_async_drop(tcx, local_decls, param_env, local) + if let ty::Adt(def, args) = ty.kind() { + if def.is_struct() && Some(def.did()) != tcx.lang_items().manually_drop() { + return def.variant(FIRST_VARIANT).fields.iter().any(|f| { + let field_ty = tcx.normalize_erasing_regions(param_env, f.ty(tcx, args)); + field_ty.is_async_drop(tcx, param_env) + }); + } + } + false } fn is_async_drop(&self, local: Local) -> bool { Self::is_async_drop_impl(self.tcx, &self.local_decls, self.param_env, local) diff --git a/compiler/rustc_mir_dataflow/Cargo.toml b/compiler/rustc_mir_dataflow/Cargo.toml index 7199db677c464..9b35c7dca837b 100644 --- a/compiler/rustc_mir_dataflow/Cargo.toml +++ b/compiler/rustc_mir_dataflow/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] # tidy-alphabetical-start +itertools = "0.11" polonius-engine = "0.13.0" regex = "1" rustc_ast = { path = "../rustc_ast" } diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs index 721a2f40fa0e6..83d6030ba7db1 100644 --- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs +++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs @@ -215,14 +215,41 @@ where // * #3:drop_term_bb: drop (obj, fut, ...) // We keep async drop unexpanded to poll-loop here, to expand it later, at StateTransform - // into states expand. - fn build_async_drop(&mut self, bb: BasicBlock) { + fn build_async_drop( + &mut self, + bb: Option, + succ: BasicBlock, + unwind: Unwind, + dropline: Option, + ) -> BasicBlock { let drop_ty = self.place_ty(self.place); let tcx = self.tcx(); let span = self.source_info.span; - let pin_obj_bb = bb; - // Resolving AsyncDrop::drop() impl function for drop_ty - let trait_ref = ty::TraitRef::from_lang_item(tcx, LangItem::AsyncDrop, span, [drop_ty]); + let pin_obj_bb = bb.unwrap_or_else( + || self.elaborator.patch().new_block(BasicBlockData { + statements: vec![], + terminator: Some(Terminator { // Temporary terminator, will be replaced by patch + source_info: self.source_info, kind: TerminatorKind::Return }), + is_cleanup: false, + })); + + // Resolving async_drop_in_place function for drop_ty + let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, Some(span)); + let trait_args = tcx.mk_args(&[drop_ty.into()]); + let sig = tcx.fn_sig(drop_fn_def_id).instantiate(tcx, trait_args); + let sig = tcx.instantiate_bound_regions_with_erased(sig); + println!("Async drop fn sig: {:?}", sig); + + /*let instance = Instance::resolve_async_drop_in_place(tcx, drop_ty); + let fn_abi = tcx.fn_abi_of_instance( + self.elaborator.param_env().and((instance, ty::List::empty())) + ).map_err(|err| { + bug!("async_drop_in_place abi error"); + }).unwrap(); + println!("Async drop abi: {:?}", fn_abi);*/ + + /*let trait_ref = ty::TraitRef::from_lang_item(tcx, LangItem::AsyncDrop, span, [drop_ty]); let (drop_trait, trait_args) = match tcx.codegen_select_candidate((ty::ParamEnv::reveal_all(), trait_ref)) { Ok(traits::ImplSource::UserDefined(traits::ImplSourceUserDefinedData { @@ -237,14 +264,19 @@ where let drop_fn_def_id = tcx.associated_item_def_ids(drop_trait)[0]; let drop_fn = Ty::new_fn_def(tcx, drop_fn_def_id, trait_args); let sig = drop_fn.fn_sig(tcx); - let sig = tcx.instantiate_bound_regions_with_erased(sig); + let sig = tcx.instantiate_bound_regions_with_erased(sig);*/ let fut_ty = sig.output(); let fut = Place::from(self.new_temp(fut_ty)); + println!("Async drop fut_ty: {:?}", fut_ty); // #1:pin_obj_bb >>> obj_ref = &mut obj let obj_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, drop_ty); let obj_ref_place = Place::from(self.new_temp(obj_ref_ty)); - let term_loc = self.elaborator.body().terminator_loc(pin_obj_bb); + let term_loc = if bb.is_some() { + self.elaborator.body().terminator_loc(pin_obj_bb) + } else { + Location { block: pin_obj_bb, statement_index: 0 } + }; self.elaborator.patch().add_assign( term_loc, obj_ref_place, @@ -275,10 +307,10 @@ where self.unwind, TerminatorKind::Drop { place: self.place, - target: self.succ, - unwind: self.unwind.into_action(), + target: succ, + unwind: unwind.into_action(), replace: false, - drop: self.dropline, + drop: dropline, async_fut: Some(fut.local), }, ); @@ -287,7 +319,7 @@ where let call_drop_bb = self.new_block( self.unwind, TerminatorKind::Call { - func: Operand::function_handle(tcx, drop_fn_def_id, tcx.mk_args(trait_args), span), + func: Operand::function_handle(tcx, drop_fn_def_id, trait_args, span), args: vec![Spanned { node: Operand::Move(Place::from(pin_obj_place)), span: DUMMY_SP, @@ -319,6 +351,7 @@ where fn_span: span, }, ); + pin_obj_bb } fn build_drop(&mut self, bb: BasicBlock) { @@ -326,9 +359,9 @@ where let is_coroutine = self.elaborator.body().coroutine.is_some(); if is_coroutine && !self.elaborator.body()[bb].is_cleanup - && drop_ty.is_async_drop(self.tcx(), self.elaborator.param_env()) + && drop_ty.needs_async_drop(self.tcx(), self.elaborator.param_env()) { - self.build_async_drop(bb); + self.build_async_drop(Some(bb), self.succ, self.unwind, self.dropline); } else { self.elaborator.patch().patch_terminator( bb, @@ -421,6 +454,7 @@ where path: Option, succ: BasicBlock, unwind: Unwind, + dropline: Option, ) -> BasicBlock { if let Some(path) = path { debug!("drop_subpath: for std field {:?}", place); @@ -432,7 +466,7 @@ where place, succ, unwind, - dropline: None, + dropline, } .elaborated_drop_block() } else { @@ -444,7 +478,7 @@ where place, succ, unwind, - dropline: None, + dropline, // Using `self.path` here to condition the drop on // our own drop flag. path: self.path, @@ -459,25 +493,32 @@ where /// /// `unwind_ladder` is such a list of steps in reverse order, /// which is called if the matching step of the drop glue panics. + /// + /// `dropline_ladder` is a similar list of steps in reverse order, + /// which is called if the matching step of the drop glue will contain async drop + /// (expanded later to Yield) and the containing coroutine will be dropped at this point. fn drop_halfladder( &mut self, unwind_ladder: &[Unwind], + dropline_ladder: &[Option], mut succ: BasicBlock, fields: &[(Place<'tcx>, Option)], ) -> Vec { iter::once(succ) - .chain(fields.iter().rev().zip(unwind_ladder).map(|(&(place, path), &unwind_succ)| { - succ = self.drop_subpath(place, path, succ, unwind_succ); + .chain(itertools::izip!(fields.iter().rev(), unwind_ladder, dropline_ladder).map( + |(&(place, path), &unwind_succ, &dropline_to)| { + succ = self.drop_subpath(place, path, succ, unwind_succ, dropline_to); succ })) .collect() } - fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind) { + fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind, Option) { // Clear the "master" drop flag at the end. This is needed // because the "master" drop protects the ADT's discriminant, // which is invalidated after the ADT is dropped. - (self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind), self.unwind) + (self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind), + self.unwind, self.dropline) } /// Creates a full drop ladder, consisting of 2 connected half-drop-ladders @@ -495,6 +536,22 @@ where /// .c2: /// ELAB(drop location.2 [target=`self.unwind`]) /// + /// For possible-async drops in coroutines we also need dropline ladder + /// .d0 (mainline): + /// ELAB(drop location.0 [target=.d1, unwind=.c1, drop=.e1]) + /// .d1 (mainline): + /// ELAB(drop location.1 [target=.d2, unwind=.c2, drop=.e2]) + /// .d2 (mainline): + /// ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`, drop=`self.drop`]) + /// .c1 (unwind): + /// ELAB(drop location.1 [target=.c2]) + /// .c2 (unwind): + /// ELAB(drop location.2 [target=`self.unwind`]) + /// .e1 (dropline): + /// ELAB(drop location.1 [target=.e2, unwind=.c2]) + /// .e2 (dropline): + /// ELAB(drop location.2 [target=`self.drop`, unwind=`self.unwind`]) + /// /// NOTE: this does not clear the master drop flag, so you need /// to point succ/unwind on a `drop_ladder_bottom`. fn drop_ladder( @@ -502,8 +559,11 @@ where fields: Vec<(Place<'tcx>, Option)>, succ: BasicBlock, unwind: Unwind, - ) -> (BasicBlock, Unwind) { + dropline: Option, + ) -> (BasicBlock, Unwind, Option) { debug!("drop_ladder({:?}, {:?})", self, fields); + assert!(if unwind.is_cleanup() { dropline.is_none() } else { true }, + "Dropline is set for cleanup drop ladder"); let mut fields = fields; fields.retain(|&(place, _)| { @@ -512,17 +572,26 @@ where debug!("drop_ladder - fields needing drop: {:?}", fields); + let dropline_ladder: Vec> = vec![None; fields.len() + 1]; let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1]; - let unwind_ladder: Vec<_> = if let Unwind::To(target) = unwind { - let halfladder = self.drop_halfladder(&unwind_ladder, target, &fields); + let unwind_ladder: Vec<_> = if let Unwind::To(succ) = unwind { + let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); halfladder.into_iter().map(Unwind::To).collect() } else { unwind_ladder }; + let dropline_ladder: Vec<_> = if let Some(succ) = dropline { + let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); + halfladder.into_iter().map(Some).collect() + } else { + dropline_ladder + }; - let normal_ladder = self.drop_halfladder(&unwind_ladder, succ, &fields); + let normal_ladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); - (*normal_ladder.last().unwrap(), *unwind_ladder.last().unwrap()) + (*normal_ladder.last().unwrap(), + *unwind_ladder.last().unwrap(), + *dropline_ladder.last().unwrap()) } fn open_drop_for_tuple(&mut self, tys: &[Ty<'tcx>]) -> BasicBlock { @@ -539,8 +608,8 @@ where }) .collect(); - let (succ, unwind) = self.drop_ladder_bottom(); - self.drop_ladder(fields, succ, unwind).0 + let (succ, unwind, dropline) = self.drop_ladder_bottom(); + self.drop_ladder(fields, succ, unwind, dropline).0 } /// Drops the T contained in a `Box` if it has not been moved out of @@ -551,6 +620,7 @@ where args: GenericArgsRef<'tcx>, succ: BasicBlock, unwind: Unwind, + dropline: Option, ) -> BasicBlock { // drop glue is sent straight to codegen // box cannot be directly dereferenced @@ -566,7 +636,7 @@ where let interior_path = self.elaborator.deref_subpath(self.path); - self.drop_subpath(interior, interior_path, succ, unwind) + self.drop_subpath(interior, interior_path, succ, unwind, dropline) } #[instrument(level = "debug", ret)] @@ -589,38 +659,41 @@ where let skip_contents = adt.is_union() || Some(adt.did()) == self.tcx().lang_items().manually_drop(); let contents_drop = if skip_contents { - (self.succ, self.unwind) + (self.succ, self.unwind, self.dropline) } else { self.open_drop_for_adt_contents(adt, args) }; if adt.is_box() { // we need to drop the inside of the box before running the destructor - let succ = self.destructor_call_block(contents_drop); + let succ = self.destructor_call_block_sync((contents_drop.0, contents_drop.1)); let unwind = contents_drop .1 - .map(|unwind| self.destructor_call_block((unwind, Unwind::InCleanup))); + .map(|unwind| self.destructor_call_block_sync((unwind, Unwind::InCleanup))); + let dropline = contents_drop + .2 + .map(|dropline| self.destructor_call_block_sync((dropline, contents_drop.1))); - self.open_drop_for_box_contents(adt, args, succ, unwind) + self.open_drop_for_box_contents(adt, args, succ, unwind, dropline) } else if adt.has_dtor(self.tcx()) { self.destructor_call_block(contents_drop) } else { contents_drop.0 } } - + fn open_drop_for_adt_contents( &mut self, adt: ty::AdtDef<'tcx>, args: GenericArgsRef<'tcx>, - ) -> (BasicBlock, Unwind) { - let (succ, unwind) = self.drop_ladder_bottom(); + ) -> (BasicBlock, Unwind, Option) { + let (succ, unwind, dropline) = self.drop_ladder_bottom(); if !adt.is_enum() { let fields = self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args); - self.drop_ladder(fields, succ, unwind) + self.drop_ladder(fields, succ, unwind, dropline) } else { - self.open_drop_for_multivariant(adt, args, succ, unwind) + self.open_drop_for_multivariant(adt, args, succ, unwind, dropline) } } @@ -630,11 +703,14 @@ where args: GenericArgsRef<'tcx>, succ: BasicBlock, unwind: Unwind, - ) -> (BasicBlock, Unwind) { + dropline: Option, + ) -> (BasicBlock, Unwind, Option) { let mut values = Vec::with_capacity(adt.variants().len()); let mut normal_blocks = Vec::with_capacity(adt.variants().len()); let mut unwind_blocks = if unwind.is_cleanup() { None } else { Some(Vec::with_capacity(adt.variants().len())) }; + let mut dropline_blocks = + if dropline.is_none() { None } else { Some(Vec::with_capacity(adt.variants().len())) }; let mut have_otherwise_with_drop_glue = false; let mut have_otherwise = false; @@ -672,11 +748,15 @@ where let unwind_blocks = unwind_blocks.as_mut().unwrap(); let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1]; - let halfladder = self.drop_halfladder(&unwind_ladder, unwind, &fields); + let dropline_ladder: Vec> = vec![None; fields.len() + 1]; + let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, unwind, &fields); unwind_blocks.push(halfladder.last().cloned().unwrap()); } - let (normal, _) = self.drop_ladder(fields, succ, unwind); + let (normal, _, drop_bb) = self.drop_ladder(fields, succ, unwind, dropline); normal_blocks.push(normal); + if dropline.is_some() { + dropline_blocks.as_mut().unwrap().push(drop_bb.unwrap()); + } } else { have_otherwise = true; @@ -716,6 +796,15 @@ where Unwind::InCleanup, ) }), + dropline.map(|dropline| { + self.adt_switch_block( + adt, + dropline_blocks.unwrap(), + &values, + dropline, + unwind, + ) + }), ) } @@ -755,8 +844,8 @@ where self.drop_flag_test_block(switch_block, succ, unwind) } - fn destructor_call_block(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock { - debug!("destructor_call_block({:?}, {:?})", self, succ); + fn destructor_call_block_sync(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock { + debug!("destructor_call_block_sync({:?}, {:?})", self, succ); let tcx = self.tcx(); let drop_trait = tcx.require_lang_item(LangItem::Drop, None); let drop_fn = tcx.associated_item_def_ids(drop_trait)[0]; @@ -810,6 +899,24 @@ where self.drop_flag_test_block(destructor_block, succ, unwind) } + fn destructor_call_block( + &mut self, + (succ, unwind, dropline): (BasicBlock, Unwind, Option) + ) -> BasicBlock { + debug!("destructor_call_block({:?}, {:?})", self, succ); + let ty = self.place_ty(self.place); + if dropline.is_some() && ty.needs_async_drop(self.tcx(), self.elaborator.param_env()) { + let destructor_block = self.build_async_drop(None, succ, unwind, dropline); + + let block_start = Location { block: destructor_block, statement_index: 0 }; + self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow); + + self.drop_flag_test_block(destructor_block, succ, unwind) + } else { + self.destructor_call_block_sync((succ, unwind)) + } + } + /// Create a loop that drops an array: /// /// ```text @@ -944,8 +1051,8 @@ where (tcx.mk_place_elem(self.place, project), path) }) .collect::>(); - let (succ, unwind) = self.drop_ladder_bottom(); - return self.drop_ladder(fields, succ, unwind).0; + let (succ, unwind, dropline) = self.drop_ladder_bottom(); + return self.drop_ladder(fields, succ, unwind, dropline).0; } } @@ -1047,12 +1154,12 @@ where fn elaborated_drop_block(&mut self) -> BasicBlock { debug!("elaborated_drop_block({:?})", self); - let blk = self.drop_block(self.succ, self.unwind); + let blk = self.drop_block_simple(self.succ, self.unwind); self.elaborate_drop(blk); blk } - fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { + fn drop_block_simple(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { let block = TerminatorKind::Drop { place: self.place, target, @@ -1064,6 +1171,26 @@ where self.new_block(unwind, block) } + fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { + let drop_ty = self.place_ty(self.place); + let is_coroutine = self.elaborator.body().coroutine.is_some(); + if is_coroutine && !unwind.is_cleanup() + && drop_ty.needs_async_drop(self.tcx(), self.elaborator.param_env()) + { + self.build_async_drop(None, self.succ, unwind, self.dropline) + } else { + let block = TerminatorKind::Drop { + place: self.place, + target, + unwind: unwind.into_action(), + replace: false, + drop: self.dropline, + async_fut: None, + }; + self.new_block(unwind, block) + } + } + fn goto_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { let block = TerminatorKind::Goto { target }; self.new_block(unwind, block) diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 2d43c8f3ed417..1043869842628 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -568,6 +568,33 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } } +// Fix return Poll::Pending statement into Poll<()>::Pending for async drop function +struct FixReturnPendingVisitor<'tcx> { + tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for FixReturnPendingVisitor<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_assign( + &mut self, + place: &mut Place<'tcx>, + rvalue: &mut Rvalue<'tcx>, + _location: Location, + ) { + if place.local != RETURN_PLACE { return; } + + // Converting `_0 = Poll::::Pending` to `_0 = Poll::<()>::Pending` + if let Rvalue::Aggregate(ref mut kind, _) = rvalue { + if let AggregateKind::Adt(_, _, ref mut args, _, _) = **kind { + *args = self.tcx.mk_args(&[Ty::new_unit(self.tcx).into()]); + } + } + } +} + fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let coroutine_ty = body.local_decls.raw[1].ty; @@ -1645,6 +1672,8 @@ fn create_coroutine_drop_shim_async<'tcx>( // not a coroutine body itself; it just has its drop built out of it. let _ = body.coroutine.take(); + FixReturnPendingVisitor { tcx }.visit_body(&mut body); + // Poison the coroutine when it unwinds if can_unwind { generate_poison_block_and_redirect_unwinds_there(transform, &mut body); @@ -1722,6 +1751,64 @@ fn create_coroutine_drop_shim_async<'tcx>( body } +// Create async drop shim proxy function for future_drop_poll +// It is just { call coroutine_drop(); return Poll::Ready(); } +fn create_coroutine_drop_shim_proxy_async<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + coroutine_ty: Ty<'tcx>, +) -> Body<'tcx> { + let mut body = body.clone(); + // Take the coroutine info out of the body, since the drop shim is + // not a coroutine body itself; it just has its drop built out of it. + let _ = body.coroutine.take(); + let basic_blocks: IndexVec> = IndexVec::new(); + body.basic_blocks = BasicBlocks::new(basic_blocks); + body.var_debug_info.clear(); + + // Keeping return value and args + body.local_decls.truncate(1 + body.arg_count); + + let source_info = SourceInfo::outermost(body.span); + + // Replace the return variable: Poll to Poll<()> + let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None)); + let poll_enum = Ty::new_adt(tcx, poll_adt_ref, tcx.mk_args(&[Ty::new_unit(tcx).into()])); + body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(poll_enum, source_info); + + // call coroutine_drop() + let call_bb = body.basic_blocks_mut().push(BasicBlockData { + statements: Vec::new(), + terminator: None, + is_cleanup: false, + }); + + // return Poll::Ready() + let ret_bb = insert_poll_ready_block(tcx, &mut body); + + let kind = TerminatorKind::Drop { + place: Place::from(SELF_ARG), + target: ret_bb, + unwind: UnwindAction::Continue, + replace: false, + drop: None, + async_fut: None, + }; + body.basic_blocks_mut()[call_bb].terminator = Some(Terminator { source_info, kind }); + + let coroutine_instance = body.source.instance; + let drop_poll = tcx.require_lang_item(LangItem::FutureDropPoll, None); + let drop_instance = InstanceDef::FutureDropPollShim(drop_poll, coroutine_ty); + + // Temporary change MirSource to coroutine's instance so that dump_mir produces more sensible + // filename. + body.source.instance = coroutine_instance; + dump_mir(tcx, false, "coroutine_drop_proxy_async", &0, &body, |_, _| Ok(())); + body.source.instance = drop_instance; + + body +} + fn insert_term_block<'tcx>(body: &mut Body<'tcx>, kind: TerminatorKind<'tcx>) -> BasicBlock { let source_info = SourceInfo::outermost(body.span); body.basic_blocks_mut().push(BasicBlockData { @@ -2268,6 +2355,11 @@ impl<'tcx> MirPass<'tcx> for StateTransform { // Run derefer to fix Derefs that are not in the first place deref_finder(tcx, &mut drop_shim); body.coroutine.as_mut().unwrap().coroutine_drop = Some(drop_shim); + + // For coroutine with sync drop, generating async proxy for `future_drop_poll` call + let mut proxy_shim = create_coroutine_drop_shim_proxy_async(tcx, body, coroutine_ty); + deref_finder(tcx, &mut proxy_shim); + body.coroutine.as_mut().unwrap().coroutine_drop_proxy_async = Some(proxy_shim); } // Create the Coroutine::resume / Future::poll function diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 1df797e5ddd74..a04c4840cf4f7 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -321,7 +321,9 @@ impl<'tcx> Inliner<'tcx> { | InstanceDef::DropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::ThreadLocalShim(..) - | InstanceDef::FnPtrAddrShim(..) => return Ok(()), + | InstanceDef::FnPtrAddrShim(..) + | InstanceDef::AsyncDropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) => return Ok(()), } if self.tcx.is_constructor(callee_def_id) { @@ -1034,21 +1036,17 @@ fn try_instance_mir<'tcx>( tcx: TyCtxt<'tcx>, instance: InstanceDef<'tcx>, ) -> Result<&'tcx Body<'tcx>, &'static str> { - match instance { - ty::InstanceDef::DropGlue(_, Some(ty)) => match ty.kind() { - ty::Adt(def, args) => { - let fields = def.all_fields(); - for field in fields { - let field_ty = field.ty(tcx, args); - if field_ty.has_param() && field_ty.has_projections() { - return Err("cannot build drop shim for polymorphic type"); - } - } - - Ok(tcx.instance_mir(instance)) + if let ty::InstanceDef::DropGlue(_, Some(ty)) + | ty::InstanceDef::AsyncDropGlueCtorShim(_, Some(ty)) = instance + && let ty::Adt(def, args) = ty.kind() + { + let fields = def.all_fields(); + for field in fields { + let field_ty = field.ty(tcx, args); + if field_ty.has_param() && field_ty.has_projections() { + return Err("cannot build drop shim for polymorphic type"); } - _ => Ok(tcx.instance_mir(instance)), - }, - _ => Ok(tcx.instance_mir(instance)), + } } + Ok(tcx.instance_mir(instance)) } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 1bc5571a0b1a2..1a5d15a5e2b54 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -92,8 +92,11 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( // This shim does not call any other functions, thus there can be no recursion. InstanceDef::FnPtrAddrShim(..) => continue, - InstanceDef::DropGlue(..) | InstanceDef::FutureDropPollShim(..) => { - // FIXME: A not fully substituted drop shim can cause ICEs if one attempts to + InstanceDef::DropGlue(..) + | InstanceDef::FutureDropPollShim(..) + | InstanceDef::AsyncDropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) => { + // FIXME: A not fully instantiated drop shim can cause ICEs if one attempts to // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this // needs some more analysis. if callee.has_param() { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index c5cee33fb92d0..611b8c8197c57 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -22,6 +22,8 @@ use crate::{ use rustc_middle::mir::patch::MirPatch; use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle}; +mod async_destructor_ctor; + pub fn provide(providers: &mut Providers) { providers.mir_shims = make_shim; } @@ -70,7 +72,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' bug!("FutureDropPollShim not for coroutine type: ({:?})", instance); }; - let body = tcx.optimized_mir(*coroutine_def_id).coroutine_drop_async().unwrap(); + let body = tcx.optimized_mir(*coroutine_def_id).future_drop_poll().unwrap(); let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args); debug!("make_shim({:?}) = {:?}", instance, body); @@ -108,6 +110,13 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' ty::InstanceDef::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance), ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty), ty::InstanceDef::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty), + ty::InstanceDef::AsyncDropGlue(def_id, ty) => { + bug!("AsyncDropGlue unimplemented") + } + ty::InstanceDef::AsyncDropGlueCtorShim(def_id, ty) => { + println!("async_destructor_ctor::build_async_destructor_ctor_shim for ty={:?}", ty); + async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty) + } ty::InstanceDef::Virtual(..) => { bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance) } diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs new file mode 100644 index 0000000000000..6ac753cdacee6 --- /dev/null +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -0,0 +1,184 @@ +use rustc_hir::def_id::DefId; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_index::{Idx, IndexVec}; +use rustc_middle::mir::{ + BasicBlock, BasicBlockData, Body, Local, LocalDecl, MirSource, + Operand, Place, Rvalue, SourceInfo, Statement, StatementKind, Terminator, TerminatorKind +}; +use rustc_span::Span; +use rustc_target::spec::PanicStrategy; +use rustc_middle::mir::patch::MirPatch; +use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator}; +use super::{local_decls_for_sig, new_body, BorrowKind, MutBorrowKind, RetagKind, START_BLOCK, DropShimElaborator}; + +pub fn build_async_destructor_ctor_shim<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: DefId, + ty: Option>, +) -> Body<'tcx> { + debug!("build_async_destructor_ctor_shim(def_id={:?}, ty={:?})", def_id, ty); + + AsyncDestructorCtorShimBuilder::new(tcx, def_id, ty).build() +} + +struct AsyncDestructorCtorShimBuilder<'tcx> { + tcx: TyCtxt<'tcx>, + def_id: DefId, + self_ty: Option>, + span: Span, + source_info: SourceInfo, + param_env: ty::ParamEnv<'tcx>, + + stack: Vec>, + last_bb: BasicBlock, + top_cleanup_bb: Option, + + locals: IndexVec>, + bbs: IndexVec>, +} + +impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { + const SELF_PTR: Local = Local::from_u32(1); + const INPUT_COUNT: usize = 1; + const MAX_STACK_LEN: usize = 2; + + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Option>) -> Self { + let args = if let Some(ty) = self_ty { + tcx.mk_args(&[ty.into()]) + } else { + ty::GenericArgs::identity_for_item(tcx, def_id) + }; + let sig = tcx.fn_sig(def_id).instantiate(tcx, args); + let sig = tcx.instantiate_bound_regions_with_erased(sig); + let span = tcx.def_span(def_id); + + let source_info = SourceInfo::outermost(span); + + debug_assert_eq!(sig.inputs().len(), Self::INPUT_COUNT); + let locals = local_decls_for_sig(&sig, span); + + // Usual case: noop() + unwind resume + return + let mut bbs = IndexVec::with_capacity(3); + let param_env = tcx.param_env_reveal_all_normalized(def_id); + AsyncDestructorCtorShimBuilder { + tcx, + def_id, + self_ty, + span, + source_info, + param_env, + + stack: Vec::with_capacity(Self::MAX_STACK_LEN), + last_bb: bbs.push(BasicBlockData::new(None)), + top_cleanup_bb: match tcx.sess.panic_strategy() { + PanicStrategy::Unwind => { + // Don't drop input arg because it's just a pointer + Some(bbs.push(BasicBlockData { + statements: Vec::new(), + terminator: Some(Terminator { + source_info, + kind: TerminatorKind::UnwindResume, + }), + is_cleanup: true, + })) + } + PanicStrategy::Abort => None, + }, + + locals, + bbs, + } + } + + fn build(self) -> Body<'tcx> { + // TODO: generate constructor here, not just resume function + build_async_drop_shim(self.tcx, self.def_id, self.self_ty) + } +} + +// build_drop_shim analog for async drop glue +fn build_async_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option>) -> Body<'tcx> { + debug!("build_async_drop_shim(def_id={:?}, ty={:?})", def_id, ty); + + assert!(!matches!(ty, Some(ty) if ty.is_coroutine())); + + let args = if let Some(ty) = ty { + tcx.mk_args(&[ty.into()]) + } else { + ty::GenericArgs::identity_for_item(tcx, def_id) + }; + let sig = tcx.fn_sig(def_id).instantiate(tcx, args); + let sig = tcx.instantiate_bound_regions_with_erased(sig); + let span = tcx.def_span(def_id); + + let source_info = SourceInfo::outermost(span); + + let return_block = BasicBlock::new(1); + let mut blocks = IndexVec::with_capacity(2); + let block = |blocks: &mut IndexVec<_, _>, kind| { + blocks.push(BasicBlockData { + statements: vec![], + terminator: Some(Terminator { source_info, kind }), + is_cleanup: false, + }) + }; + block(&mut blocks, TerminatorKind::Goto { target: return_block }); + block(&mut blocks, TerminatorKind::Return); + + let source = MirSource::from_instance(ty::InstanceDef::AsyncDropGlue(def_id, ty)); + let mut body = + new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span); + + // The first argument (index 0), but add 1 for the return value. + let mut dropee_ptr = Place::from(Local::new(1 + 0)); + if tcx.sess.opts.unstable_opts.mir_emit_retag { + // We want to treat the function argument as if it was passed by `&mut`. As such, we + // generate + // ``` + // temp = &mut *arg; + // Retag(temp, FnEntry) + // ``` + // It's important that we do this first, before anything that depends on `dropee_ptr` + // has been put into the body. + let reborrow = Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + tcx.mk_place_deref(dropee_ptr), + ); + let ref_ty = reborrow.ty(&body.local_decls, tcx); + dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into(); + let new_statements = [ + StatementKind::Assign(Box::new((dropee_ptr, reborrow))), + StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)), + ]; + for s in new_statements { + body.basic_blocks_mut()[START_BLOCK] + .statements + .push(Statement { source_info, kind: s }); + } + } + + if ty.is_some() { + let patch = { + let param_env = tcx.param_env_reveal_all_normalized(def_id); + let mut elaborator = + DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, param_env }; + let dropee = tcx.mk_place_deref(dropee_ptr); + let resume_block = elaborator.patch.resume_block(); + elaborate_drops::elaborate_drop( + &mut elaborator, + source_info, + dropee, + (), + return_block, + elaborate_drops::Unwind::To(resume_block), + START_BLOCK, + None, + ); + elaborator.patch + }; + patch.apply(&mut body); + } + + body +} \ No newline at end of file diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 12200cff5a8c5..52545eef147f5 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -973,13 +973,17 @@ fn visit_instance_use<'tcx>( ty::InstanceDef::ThreadLocalShim(..) => { bug!("{:?} being reified", instance); } - ty::InstanceDef::DropGlue(_, None) => { + ty::InstanceDef::DropGlue(_, None) + | ty::InstanceDef::AsyncDropGlueCtorShim(_, None) + | ty::InstanceDef::AsyncDropGlue(_, None) => { // Don't need to emit noop drop glue if we are calling directly. if !is_direct_call { output.push(create_fn_mono_item(tcx, instance, source)); } } ty::InstanceDef::DropGlue(_, Some(_)) + | ty::InstanceDef::AsyncDropGlue(_, Some(_)) + | ty::InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) | ty::InstanceDef::FutureDropPollShim(..) | ty::InstanceDef::VTableShim(..) | ty::InstanceDef::ReifyShim(..) diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 7bc49777a84a3..396edf874246f 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -623,10 +623,12 @@ fn characteristic_def_id_of_mono_item<'tcx>( | ty::InstanceDef::Intrinsic(..) | ty::InstanceDef::FutureDropPollShim(..) | ty::InstanceDef::DropGlue(..) + | ty::InstanceDef::AsyncDropGlue(..) | ty::InstanceDef::Virtual(..) | ty::InstanceDef::CloneShim(..) | ty::InstanceDef::ThreadLocalShim(..) - | ty::InstanceDef::FnPtrAddrShim(..) => return None, + | ty::InstanceDef::FnPtrAddrShim(..) + | ty::InstanceDef::AsyncDropGlueCtorShim(..) => return None, }; // If this is a method, we want to put it into the same module as @@ -770,7 +772,10 @@ fn mono_item_visibility<'tcx>( }; let def_id = match instance.def { - InstanceDef::Item(def_id) | InstanceDef::DropGlue(def_id, Some(_)) => def_id, + InstanceDef::Item(def_id) + | InstanceDef::DropGlue(def_id, Some(_)) + | InstanceDef::AsyncDropGlue(def_id, Some(_)) + | InstanceDef::AsyncDropGlueCtorShim(def_id, Some(_)) => def_id, // We match the visibility of statics here InstanceDef::ThreadLocalShim(def_id) => { @@ -786,6 +791,8 @@ fn mono_item_visibility<'tcx>( | InstanceDef::ClosureOnceShim { .. } | InstanceDef::FutureDropPollShim(..) | InstanceDef::DropGlue(..) + | InstanceDef::AsyncDropGlue(..) + | InstanceDef::AsyncDropGlueCtorShim(..) | InstanceDef::CloneShim(..) | InstanceDef::FnPtrAddrShim(..) => return Visibility::Hidden, }; diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs index 94a1fb33f99c5..6ddbde345044e 100644 --- a/compiler/rustc_smir/src/rustc_smir/context.rs +++ b/compiler/rustc_smir/src/rustc_smir/context.rs @@ -403,6 +403,12 @@ impl<'tcx> Context for TablesWrapper<'tcx> { matches!(instance.def, ty::InstanceDef::DropGlue(_, None)) } + fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool { + let tables = self.0.borrow_mut(); + let instance = tables.instances[def]; + matches!(instance.def, ty::InstanceDef::AsyncDropGlueCtorShim(_, None)) + } + fn mono_instance(&self, def_id: stable_mir::DefId) -> stable_mir::mir::mono::Instance { let mut tables = self.0.borrow_mut(); let def_id = tables[def_id]; diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs index cfd6bface695c..81a8ddff5983c 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs @@ -803,7 +803,11 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> { | ty::InstanceDef::FutureDropPollShim(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) - | ty::InstanceDef::FnPtrShim(..) => stable_mir::mir::mono::InstanceKind::Shim, + | ty::InstanceDef::FnPtrShim(..) + | ty::InstanceDef::AsyncDropGlue(..) + | ty::InstanceDef::AsyncDropGlueCtorShim(..) => { + stable_mir::mir::mono::InstanceKind::Shim + } }; stable_mir::mir::mono::Instance { def, kind } } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 2e97efdd5a934..08012ff3ee895 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -431,6 +431,7 @@ symbols! { async_closure, async_drop, async_drop_in_place, + async_drop_in_place_poll, async_fn, async_fn_in_trait, async_fn_mut, diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 4a5f58443bc29..eee99f4efd70b 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -55,7 +55,9 @@ pub(super) fn mangle<'tcx>( printer .print_def_path( def_id, - if let ty::InstanceDef::DropGlue(_, _) = instance.def { + if let ty::InstanceDef::DropGlue(_, _) | ty::InstanceDef::AsyncDropGlueCtorShim(_, _) = + instance.def + { // Add the name of the dropped type to the symbol name &*instance.args } else { diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 30ca0f9d897c6..637521a573edd 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -59,6 +59,27 @@ fn resolve_instance<'tcx>( debug!(" => trivial drop glue"); ty::InstanceDef::DropGlue(def_id, None) } + } else if Some(def_id) == tcx.lang_items().async_drop_in_place_fn() { + let ty = args.type_at(0); + + if ty.needs_async_drop(tcx, param_env) { + match *ty.kind() { + ty::Closure(..) + | ty::Coroutine(..) + | ty::Tuple(..) + | ty::Adt(..) + | ty::Dynamic(..) + | ty::Array(..) + | ty::Slice(..) => {} + // Async destructor ctor shims can only be built from ADTs. + _ => return Ok(None), + } + debug!(" => nontrivial async drop glue ctor"); + ty::InstanceDef::AsyncDropGlueCtorShim(def_id, Some(ty)) + } else { + debug!(" => trivial async drop glue ctor"); + ty::InstanceDef::AsyncDropGlueCtorShim(def_id, None) + } } else if Some(def_id) == tcx.lang_items().future_drop_poll_fn() { let ty = args.type_at(0); ty::InstanceDef::FutureDropPollShim(def_id, ty) diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 08e5476ae43b7..5d35628a8ef9f 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -19,7 +19,7 @@ fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx> // needs drop. let adt_has_dtor = |adt_def: ty::AdtDef<'tcx>| adt_def.destructor(tcx).map(|_| DtorType::Significant); - let res = drop_tys_helper(tcx, query.value, query.param_env, adt_has_dtor, false) + let res = drop_tys_helper::(tcx, query.value, query.param_env, adt_has_dtor, false) .filter(filter_array_elements(tcx, query.param_env)) .next() .is_some(); @@ -28,6 +28,21 @@ fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx> res } +fn needs_async_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + // If we don't know a type doesn't need async drop, for example if it's a + // type parameter without a `Copy` bound, then we conservatively return that + // it needs async drop. + let adt_has_async_dtor = + |adt_def: ty::AdtDef<'tcx>| adt_def.async_destructor(tcx).map(|_| DtorType::Significant); + let res = drop_tys_helper::(tcx, query.value, query.param_env, adt_has_async_dtor, false) + .filter(filter_array_elements_async(tcx, query.param_env)) + .next() + .is_some(); + + debug!("needs_async_drop_raw({:?}) = {:?}", query, res); + res +} + /// HACK: in order to not mistakenly assume that `[PhantomData; N]` requires drop glue /// we check the element type for drop glue. The correct fix would be looking at the /// entirety of the code around `needs_drop_components` and this file and come up with @@ -44,12 +59,24 @@ fn filter_array_elements<'tcx>( Err(AlwaysRequiresDrop) => true, } } +fn filter_array_elements_async<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, +) -> impl Fn(&Result, AlwaysRequiresDrop>) -> bool { + move |ty| match ty { + Ok(ty) => match *ty.kind() { + ty::Array(elem, _) => tcx.needs_async_drop_raw(param_env.and(elem)), + _ => true, + }, + Err(AlwaysRequiresDrop) => true, + } +} fn has_significant_drop_raw<'tcx>( tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, ) -> bool { - let res = drop_tys_helper( + let res = drop_tys_helper::( tcx, query.value, query.param_env, @@ -63,7 +90,7 @@ fn has_significant_drop_raw<'tcx>( res } -struct NeedsDropTypes<'tcx, F> { +struct NeedsDropTypes<'tcx, F, const IS_ASYNC: bool> { tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, // Whether to reveal coroutine witnesses, this is set @@ -80,7 +107,7 @@ struct NeedsDropTypes<'tcx, F> { adt_components: F, } -impl<'tcx, F> NeedsDropTypes<'tcx, F> { +impl<'tcx, F, const IS_ASYNC: bool> NeedsDropTypes<'tcx, F, IS_ASYNC> { fn new( tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, @@ -102,7 +129,7 @@ impl<'tcx, F> NeedsDropTypes<'tcx, F> { } } -impl<'tcx, F, I> Iterator for NeedsDropTypes<'tcx, F> +impl<'tcx, F, const IS_ASYNC: bool, I> Iterator for NeedsDropTypes<'tcx, F, IS_ASYNC> where F: Fn(ty::AdtDef<'tcx>, GenericArgsRef<'tcx>) -> NeedsDropResult, I: Iterator>, @@ -116,6 +143,7 @@ where if !self.recursion_limit.value_within_limit(level) { // Not having a `Span` isn't great. But there's hopefully some other // recursion limit error as well. + println!("Needs drop recursion limit hit"); tcx.dcx().emit_err(NeedsDropOverflow { query_ty: self.query_ty }); return Some(Err(AlwaysRequiresDrop)); } @@ -145,11 +173,17 @@ where // for the coroutine witness and check whether any of the contained types // need to be dropped, and only require the captured types to be live // if they do. - ty::Coroutine(_, args) => { + ty::Coroutine(def_id, args) => { if self.reveal_coroutine_witnesses { queue_type(self, args.as_coroutine().witness()); } else { - return Some(Err(AlwaysRequiresDrop)); + if IS_ASYNC { + if tcx.optimized_mir(def_id).coroutine_requires_async_drop() { + return Some(Err(AlwaysRequiresDrop)); + } + } else { + return Some(Err(AlwaysRequiresDrop)); + } } } ty::CoroutineWitness(def_id, args) => { @@ -245,7 +279,7 @@ enum DtorType { // Depending on the implantation of `adt_has_dtor`, it is used to check if the // ADT has a destructor or if the ADT only has a significant destructor. For // understanding significant destructor look at `adt_significant_drop_tys`. -fn drop_tys_helper<'tcx>( +fn drop_tys_helper<'tcx, const IS_ASYNC: bool>( tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, param_env: rustc_middle::ty::ParamEnv<'tcx>, @@ -311,7 +345,7 @@ fn drop_tys_helper<'tcx>( .map(|v| v.into_iter()) }; - NeedsDropTypes::new(tcx, param_env, ty, adt_components) + NeedsDropTypes::<_, IS_ASYNC>::new(tcx, param_env, ty, adt_components) } fn adt_consider_insignificant_dtor<'tcx>( @@ -346,7 +380,7 @@ fn adt_drop_tys<'tcx>( let adt_has_dtor = |adt_def: ty::AdtDef<'tcx>| adt_def.destructor(tcx).map(|_| DtorType::Significant); // `tcx.type_of(def_id)` identical to `tcx.make_adt(def, identity_args)` - drop_tys_helper( + drop_tys_helper::( tcx, tcx.type_of(def_id).instantiate_identity(), tcx.param_env(def_id), @@ -356,6 +390,26 @@ fn adt_drop_tys<'tcx>( .collect::, _>>() .map(|components| tcx.mk_type_list(&components)) } + +fn adt_async_drop_tys<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: DefId, +) -> Result<&ty::List>, AlwaysRequiresDrop> { + // This is for the "adt_async_drop_tys" query, that considers all `AsyncDrop` impls. + let adt_has_dtor = + |adt_def: ty::AdtDef<'tcx>| adt_def.async_destructor(tcx).map(|_| DtorType::Significant); + // `tcx.type_of(def_id)` identical to `tcx.make_adt(def, identity_args)` + drop_tys_helper::( + tcx, + tcx.type_of(def_id).instantiate_identity(), + tcx.param_env(def_id), + adt_has_dtor, + false, + ) + .collect::, _>>() + .map(|components| tcx.mk_type_list(&components)) +} + // If `def_id` refers to a generic ADT, the queries above and below act as if they had been handed // a `tcx.make_ty(def, identity_args)` and as such it is legal to substitute the generic parameters // of the ADT into the outputted `ty`s. @@ -363,7 +417,7 @@ fn adt_significant_drop_tys( tcx: TyCtxt<'_>, def_id: DefId, ) -> Result<&ty::List>, AlwaysRequiresDrop> { - drop_tys_helper( + drop_tys_helper::( tcx, tcx.type_of(def_id).instantiate_identity(), // identical to `tcx.make_adt(def, identity_args)` tcx.param_env(def_id), @@ -377,8 +431,10 @@ fn adt_significant_drop_tys( pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { needs_drop_raw, + needs_async_drop_raw, has_significant_drop_raw, adt_drop_tys, + adt_async_drop_tys, adt_significant_drop_tys, ..*providers }; diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs index fb83dae571423..e429e8335cf7c 100644 --- a/compiler/stable_mir/src/compiler_interface.rs +++ b/compiler/stable_mir/src/compiler_interface.rs @@ -139,6 +139,9 @@ pub trait Context { /// Check if this is an empty DropGlue shim. fn is_empty_drop_shim(&self, def: InstanceDef) -> bool; + /// Check if this is an empty AsyncDropGlueCtor shim. + fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool; + /// Convert a non-generic crate item into an instance. /// This function will panic if the item is generic. fn mono_instance(&self, def_id: DefId) -> Instance; diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs index 6c791ae855225..771486538a005 100644 --- a/compiler/stable_mir/src/mir/mono.rs +++ b/compiler/stable_mir/src/mir/mono.rs @@ -136,7 +136,10 @@ impl Instance { /// When generating code for a Drop terminator, users can ignore an empty drop glue. /// These shims are only needed to generate a valid Drop call done via VTable. pub fn is_empty_shim(&self) -> bool { - self.kind == InstanceKind::Shim && with(|cx| cx.is_empty_drop_shim(self.def)) + self.kind == InstanceKind::Shim + && with(|cx| { + cx.is_empty_drop_shim(self.def) || cx.is_empty_async_drop_ctor_shim(self.def) + }) } /// Try to constant evaluate the instance into a constant with the given type. diff --git a/library/core/src/future/async_drop.rs b/library/core/src/future/async_drop.rs index c0b650f9ed624..a73211f11bb11 100644 --- a/library/core/src/future/async_drop.rs +++ b/library/core/src/future/async_drop.rs @@ -52,6 +52,7 @@ where future_drop_poll(fut, cx) } +#[cfg(not(bootstrap))] #[unstable(feature = "async_drop", issue = "none")] #[cfg(all(not(no_global_oom_handling), not(test)))] #[lang = "drop_in_place_future"] @@ -62,3 +63,9 @@ where // FIXME: Support poll loop for async drop of future (future_drop_poll) panic!("Unimplemented drop_in_place_future"); } + +#[cfg(not(bootstrap))] +#[unstable(feature = "async_drop", issue = "none")] +#[lang = "async_drop_in_place"] +pub async unsafe fn async_drop_in_place(to_drop: *mut T) { +} diff --git a/tests/ui/async-await/async-drop/async-drop-open.rs b/tests/ui/async-await/async-drop/async-drop-open.rs new file mode 100644 index 0000000000000..082888868ea67 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-open.rs @@ -0,0 +1,128 @@ +// run-pass +// check-run-results +// struct `Foo` has both sync and async drop. +// Struct `Complex` contains three `Foo` fields and one of them is moved out. + +#![feature(async_drop)] +#![allow(incomplete_features)] + +use std::mem::ManuallyDrop; + +// edition: 2021 + +#[inline(never)] +fn myprintln(msg: &str, my_resource_handle: usize) { + println!("{} : {}", msg, my_resource_handle); +} + +use std::{ + future::{Future, AsyncDrop}, + pin::{pin, Pin}, + sync::{mpsc, Arc}, + task::{Context, Poll, Wake, Waker}, +}; + +use std::future::future_drop_poll; + +struct Foo { + my_resource_handle: usize, +} + +#[allow(dead_code)] +struct Complex { + field1: Foo, + field2: Foo, + field3: Foo, +} + +impl Complex { + fn new(my_resource_handle: usize) -> Self { + myprintln("Complex::new()", my_resource_handle); + let field1 = Foo::new(my_resource_handle); + let field2 = Foo::new(my_resource_handle + 1); + let field3 = Foo::new(my_resource_handle + 2); + Complex { field1, field2, field3 } + } +} + +impl Foo { + fn new(my_resource_handle: usize) -> Self { + let out = Foo { + my_resource_handle, + }; + myprintln("Foo::new()", my_resource_handle); + out + } +} + +impl Drop for Foo { + fn drop(&mut self) { + myprintln("Foo::drop()", self.my_resource_handle); + } +} + +impl AsyncDrop for Foo { + async fn drop(self: Pin<&mut Self>) { + myprintln("Foo::async drop()", self.my_resource_handle); + } +} + +fn main() { + { + let _ = Foo::new(7); + } + println!("Middle"); + // Inside field1 and field3 of Complex must be dropped (as async drop) + // field2 must be dropped here (as sync drop) + { + let _field2 = block_on(bar(10)); + } + println!("Done") +} + +async fn bar(ident_base: usize) -> Foo { + let complex = Complex::new(ident_base); + complex.field2 +} + +fn block_on(fut_unpin: F) -> F::Output +where + F: Future, +{ + let mut fut_pin = pin!(ManuallyDrop::new(fut_unpin)); + let mut fut: Pin<&mut F> = unsafe { + Pin::map_unchecked_mut(fut_pin.as_mut(), |x| &mut **x) + }; + let (waker, rx) = simple_waker(); + let mut context = Context::from_waker(&waker); + let rv = loop { + match fut.as_mut().poll(&mut context) { + Poll::Ready(out) => break out, + // expect wake in polls + Poll::Pending => rx.try_recv().unwrap(), + } + }; + loop { + match future_drop_poll(fut.as_mut(), &mut context) { + Poll::Ready(()) => break, + // expect wake in polls + Poll::Pending => rx.try_recv().unwrap(), + } + } + rv +} + +fn simple_waker() -> (Waker, mpsc::Receiver<()>) { + struct SimpleWaker { + tx: std::sync::mpsc::Sender<()>, + } + + impl Wake for SimpleWaker { + fn wake(self: Arc) { + self.tx.send(()).unwrap(); + } + } + + let (tx, rx) = mpsc::channel(); + (Waker::from(Arc::new(SimpleWaker { tx })), rx) +} diff --git a/tests/ui/async-await/async-drop/async-drop-open.run.stdout b/tests/ui/async-await/async-drop/async-drop-open.run.stdout new file mode 100644 index 0000000000000..ce67e661176d0 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-open.run.stdout @@ -0,0 +1,11 @@ +Foo::new() : 7 +Foo::drop() : 7 +Middle +Complex::new() : 10 +Foo::new() : 10 +Foo::new() : 11 +Foo::new() : 12 +Foo::drop() : 10 +Foo::drop() : 12 +Foo::drop() : 11 +Done