diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index d2f11d48140c9..41201f6603d96 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -364,7 +364,7 @@ fn exported_symbols_provider_local( )); } MonoItem::Fn(Instance { - def: InstanceKind::AsyncDropGlueCtorShim(def_id, Some(ty)), + def: InstanceKind::AsyncDropGlueCtorShim(def_id, ty), args, }) => { // A little sanity-check @@ -381,6 +381,16 @@ fn exported_symbols_provider_local( }, )); } + MonoItem::Fn(Instance { def: InstanceKind::AsyncDropGlue(_, ty), args: _ }) => { + symbols.push(( + ExportedSymbol::AsyncDropGlue(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } _ => { // Any other symbols don't qualify for sharing } @@ -404,6 +414,7 @@ fn upstream_monomorphizations_provider( let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn(); let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn(); + let async_drop_in_place_poll_fn_def_id = tcx.lang_items().async_drop_in_place_poll_fn(); for &cnum in cnums.iter() { for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { @@ -422,8 +433,13 @@ fn upstream_monomorphizations_provider( if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id { (async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()])) } else { - // `drop_in_place` in place does not exist, don't try - // to use it. + continue; + } + } + ExportedSymbol::AsyncDropGlue(ty) => { + if let Some(poll_fn_def_id) = async_drop_in_place_poll_fn_def_id { + (poll_fn_def_id, tcx.mk_args(&[ty.into()])) + } else { continue; } } @@ -575,6 +591,13 @@ pub fn symbol_name_for_instance_in_crate<'tcx>( instantiating_crate, ) } + ExportedSymbol::AsyncDropGlue(ty) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::resolve_async_drop_in_place_poll(tcx, ty), + instantiating_crate, + ) + } ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), } } @@ -626,6 +649,7 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>( // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the // target's default symbol decoration scheme. ExportedSymbol::AsyncDropGlueCtorShim(..) => None, + ExportedSymbol::AsyncDropGlue(..) => None, // NoDefId always follow the target's default symbol decoration scheme. ExportedSymbol::NoDefId(..) => None, // ThreadLocalShim always follow the target's default symbol decoration scheme. diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index 568a9a3a637be..d4c558125802d 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -552,6 +552,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { | ty::InstanceKind::FnPtrAddrShim(..) | ty::InstanceKind::ThreadLocalShim(..) | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::AsyncDropGlue(..) + | ty::InstanceKind::FutureDropPollShim(..) | ty::InstanceKind::Item(_) => { // We need MIR for this fn let Some((body, instance)) = M::find_mir_or_eval_fn( diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index f4ac6156919fa..83e6a83fedf0d 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -182,6 +182,7 @@ language_item_table! { AsyncDrop, sym::async_drop, async_drop_trait, Target::Trait, GenericRequirement::None; AsyncDropInPlace, sym::async_drop_in_place, async_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1); AsyncDropInPlacePoll, sym::async_drop_in_place_poll, async_drop_in_place_poll_fn, Target::Closure, GenericRequirement::Exact(1); + FutureDropPoll, sym::future_drop_poll, future_drop_poll_fn, Target::Fn, GenericRequirement::Exact(1); CoerceUnsized, sym::coerce_unsized, coerce_unsized_trait, Target::Trait, GenericRequirement::Minimum(1); DispatchFromDyn, sym::dispatch_from_dyn, dispatch_from_dyn_trait, Target::Trait, GenericRequirement::Minimum(1); diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 0bfbd39879747..0a4c028f5918e 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -44,6 +44,7 @@ pub enum ExportedSymbol<'tcx> { Generic(DefId, GenericArgsRef<'tcx>), DropGlue(Ty<'tcx>), AsyncDropGlueCtorShim(Ty<'tcx>), + AsyncDropGlue(Ty<'tcx>), ThreadLocalShim(DefId), NoDefId(ty::SymbolName<'tcx>), } @@ -63,6 +64,9 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::AsyncDropGlueCtorShim(ty) => { tcx.symbol_name(ty::Instance::resolve_async_drop_in_place(tcx, ty)) } + ExportedSymbol::AsyncDropGlue(ty) => { + tcx.symbol_name(ty::Instance::resolve_async_drop_in_place_poll(tcx, ty)) + } ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance { def: ty::InstanceKind::ThreadLocalShim(def_id), args: ty::GenericArgs::empty(), diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 081a23b6ff317..2862daec51639 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -205,7 +205,13 @@ pub struct CoroutineInfo<'tcx> { /// Coroutine drop glue. This field is populated after the state transform pass. pub coroutine_drop: Option>, - /// The layout of a coroutine. This field is populated after the state transform pass. + /// Coroutine async drop glue. + pub coroutine_drop_async: Option>, + + /// When coroutine has sync drop, this is async proxy calling `coroutine_drop` sync impl. + pub coroutine_drop_proxy_async: Option>, + + /// The layout of a coroutine. Produced by the state transformation. pub coroutine_layout: Option>, /// If this is a coroutine then record the type of source expression that caused this coroutine @@ -225,6 +231,8 @@ impl<'tcx> CoroutineInfo<'tcx> { yield_ty: Some(yield_ty), resume_ty: Some(resume_ty), coroutine_drop: None, + coroutine_drop_async: None, + coroutine_drop_proxy_async: None, coroutine_layout: None, } } @@ -590,6 +598,26 @@ impl<'tcx> Body<'tcx> { self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref()) } + #[inline] + pub fn coroutine_drop_async(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop_async.as_ref()) + } + + #[inline] + pub fn coroutine_requires_async_drop(&self) -> bool { + self.coroutine_drop_async().is_some() + } + + #[inline] + pub fn future_drop_poll(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref().and_then(|coroutine| { + coroutine + .coroutine_drop_async + .as_ref() + .or(coroutine.coroutine_drop_proxy_async.as_ref()) + }) + } + #[inline] pub fn coroutine_kind(&self) -> Option { self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind) diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 69b5e0a79c7fa..c7cbeaf959752 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -71,6 +71,8 @@ impl<'tcx> MonoItem<'tcx> { // statements, plus one for the terminator. InstanceKind::Item(..) | InstanceKind::DropGlue(..) + | InstanceKind::FutureDropPollShim(..) + | InstanceKind::AsyncDropGlue(..) | InstanceKind::AsyncDropGlueCtorShim(..) => { let mir = tcx.instance_mir(instance.def); mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum() @@ -419,6 +421,8 @@ impl<'tcx> CodegenUnit<'tcx> { | InstanceKind::CloneShim(..) | InstanceKind::ThreadLocalShim(..) | InstanceKind::FnPtrAddrShim(..) + | InstanceKind::AsyncDropGlue(..) + | InstanceKind::FutureDropPollShim(..) | InstanceKind::AsyncDropGlueCtorShim(..) => None, } } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index cf1cb9a776760..90987cca409dc 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -232,9 +232,20 @@ fn dump_path<'tcx>( })); s } - ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => { - // Unfortunately, pretty-printed typed are not very filename-friendly. - // We dome some filtering. + ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => { + let mut s = ".".to_owned(); + s.extend(ty.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s + } + ty::InstanceKind::AsyncDropGlue(_, ty) => { + let ty::Coroutine(_, args) = ty.kind() else { + bug!(); + }; + let ty = args.first().unwrap().expect_ty(); let mut s = ".".to_owned(); s.extend(ty.to_string().chars().filter_map(|c| match c { ' ' => None, @@ -243,6 +254,21 @@ fn dump_path<'tcx>( })); s } + ty::InstanceKind::FutureDropPollShim(_, proxy_cor, impl_cor) => { + let mut s = ".".to_owned(); + s.extend(proxy_cor.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s.push_str("."); + s.extend(impl_cor.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s + } _ => String::new(), }; diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 7861ac9296055..4956097191c49 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -349,17 +349,21 @@ macro_rules! make_mir_visitor { coroutine_closure_def_id: _def_id, receiver_by_ref: _, } | - ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) | ty::InstanceKind::DropGlue(_def_id, None) => {} ty::InstanceKind::FnPtrShim(_def_id, ty) | ty::InstanceKind::DropGlue(_def_id, Some(ty)) | ty::InstanceKind::CloneShim(_def_id, ty) | ty::InstanceKind::FnPtrAddrShim(_def_id, ty) | - ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, Some(ty)) => { + ty::InstanceKind::AsyncDropGlue(_def_id, ty) | + ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, ty) => { // FIXME(eddyb) use a better `TyContext` here. self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); } + ty::InstanceKind::FutureDropPollShim(_def_id, proxy_ty, impl_ty) => { + self.visit_ty($(& $mutability)? *proxy_ty, TyContext::Location(location)); + self.visit_ty($(& $mutability)? *impl_ty, TyContext::Location(location)); + } } self.visit_args(callee_args, location); } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f81f50340ee00..a516f00e292da 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1562,6 +1562,10 @@ impl<'tcx> TyCtxt<'tcx> { self.coroutine_kind(def_id).is_some() } + pub fn is_templated_coroutine(self, def_id: DefId) -> bool { + Some(def_id) == self.lang_items().async_drop_in_place_poll_fn() + } + /// Returns the movability of the coroutine of `def_id`, or panics /// if given a `def_id` that is not a coroutine. pub fn coroutine_movability(self, def_id: DefId) -> hir::Movability { diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 072951d131917..07218f1668bba 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -146,6 +146,9 @@ pub enum InstanceKind<'tcx> { /// native support. ThreadLocalShim(DefId), + /// Proxy shim for async drop of future (def_id, proxy_cor_ty, impl_cor_ty) + FutureDropPollShim(DefId, Ty<'tcx>, Ty<'tcx>), + /// `core::ptr::drop_in_place::`. /// /// The `DefId` is for `core::ptr::drop_in_place`. @@ -172,7 +175,13 @@ pub enum InstanceKind<'tcx> { /// /// The `DefId` is for `core::future::async_drop::async_drop_in_place`, the `Ty` /// is the type `T`. - AsyncDropGlueCtorShim(DefId, Option>), + AsyncDropGlueCtorShim(DefId, Ty<'tcx>), + + /// `core::future::async_drop::async_drop_in_place::<'_, T>::{closure}`. + /// + /// async_drop_in_place poll function implementation (for generated coroutine). + /// `Ty` here is `async_drop_in_place::{closure}` coroutine type, not just `T` + AsyncDropGlue(DefId, Ty<'tcx>), } impl<'tcx> Instance<'tcx> { @@ -216,7 +225,9 @@ impl<'tcx> Instance<'tcx> { .upstream_monomorphizations_for(def) .and_then(|monos| monos.get(&self.args).cloned()), InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args), - InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => { + InstanceKind::AsyncDropGlue(_, _) => None, + InstanceKind::FutureDropPollShim(_, _, _) => None, + InstanceKind::AsyncDropGlueCtorShim(_, _) => { tcx.upstream_async_drop_glue_for(self.args) } _ => None, @@ -243,6 +254,8 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::DropGlue(def_id, _) | InstanceKind::CloneShim(def_id, _) | InstanceKind::FnPtrAddrShim(def_id, _) + | InstanceKind::FutureDropPollShim(def_id, _, _) + | InstanceKind::AsyncDropGlue(def_id, _) | InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id, } } @@ -252,7 +265,9 @@ impl<'tcx> InstanceKind<'tcx> { match self { ty::InstanceKind::Item(def) => Some(def), ty::InstanceKind::DropGlue(def_id, Some(_)) - | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_)) + | InstanceKind::AsyncDropGlueCtorShim(def_id, _) + | InstanceKind::AsyncDropGlue(def_id, _) + | InstanceKind::FutureDropPollShim(def_id, ..) | InstanceKind::ThreadLocalShim(def_id) => Some(def_id), InstanceKind::VTableShim(..) | InstanceKind::ReifyShim(..) @@ -262,7 +277,6 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::ClosureOnceShim { .. } | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } | InstanceKind::DropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) | InstanceKind::CloneShim(..) | InstanceKind::FnPtrAddrShim(..) => None, } @@ -287,7 +301,9 @@ impl<'tcx> InstanceKind<'tcx> { let def_id = match *self { ty::InstanceKind::Item(def) => def, ty::InstanceKind::DropGlue(_, Some(_)) => return false, - ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => return false, + ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => return ty.is_coroutine(), + ty::InstanceKind::FutureDropPollShim(_, _, _) => return false, + ty::InstanceKind::AsyncDropGlue(_, _) => return false, ty::InstanceKind::ThreadLocalShim(_) => return false, _ => return true, }; @@ -309,7 +325,7 @@ impl<'tcx> InstanceKind<'tcx> { return true; } if let ty::InstanceKind::DropGlue(.., Some(ty)) - | ty::InstanceKind::AsyncDropGlueCtorShim(.., Some(ty)) = *self + | ty::InstanceKind::AsyncDropGlueCtorShim(.., ty) = *self { // Drop glue generally wants to be instantiated at every codegen // unit, but without an #[inline] hint. We should make this @@ -364,11 +380,12 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::FnPtrAddrShim(..) | InstanceKind::FnPtrShim(..) | InstanceKind::DropGlue(_, Some(_)) - | InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => false, + | InstanceKind::FutureDropPollShim(..) + | InstanceKind::AsyncDropGlue(_, _) => false, + InstanceKind::AsyncDropGlueCtorShim(_, _) => false, InstanceKind::ClosureOnceShim { .. } | InstanceKind::ConstructCoroutineInClosureShim { .. } | InstanceKind::DropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) | InstanceKind::Item(_) | InstanceKind::Intrinsic(..) | InstanceKind::ReifyShim(..) @@ -445,8 +462,11 @@ pub fn fmt_instance( InstanceKind::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), InstanceKind::CloneShim(_, ty) => write!(f, " - shim({ty})"), InstanceKind::FnPtrAddrShim(_, ty) => write!(f, " - shim({ty})"), - InstanceKind::AsyncDropGlueCtorShim(_, None) => write!(f, " - shim(None)"), - InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), + InstanceKind::FutureDropPollShim(_, proxy_ty, impl_ty) => { + write!(f, " - dropshim({proxy_ty}-{impl_ty})") + } + InstanceKind::AsyncDropGlue(_, ty) => write!(f, " - shim({ty})"), + InstanceKind::AsyncDropGlueCtorShim(_, ty) => write!(f, " - shim(Some({ty}))"), } } @@ -464,6 +484,59 @@ impl<'tcx> fmt::Display for Instance<'tcx> { } } +// async_drop_in_place::coroutine.poll, when T is a standart coroutine, +// should be resolved to this coroutine's future_drop_poll (through FutureDropPollShim proxy). +// async_drop_in_place::coroutine>::coroutine.poll, +// when T is a standart coroutine, should be resolved to this coroutine's future_drop_poll. +// async_drop_in_place::coroutine>::coroutine.poll, +// when T is not a coroutine, should be resolved to the innermost +// async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy) +fn resolve_async_drop_poll<'tcx>(tcx: TyCtxt<'tcx>, mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { + let first_cor = cor_ty; + let ty::Coroutine(_, proxy_args) = first_cor.kind() else { + bug!(); + }; + let async_drop_in_place_poll = tcx.lang_items().async_drop_in_place_poll_fn().unwrap(); + let mut child_ty = cor_ty; + loop { + if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + cor_ty = child_ty; + if *child_def == async_drop_in_place_poll { + child_ty = child_args.first().unwrap().expect_ty(); + continue; + } else { + return Instance { + def: ty::InstanceKind::FutureDropPollShim( + async_drop_in_place_poll, + first_cor, + cor_ty, + ), + args: proxy_args, + }; + } + } else { + let ty::Coroutine(_, child_args) = cor_ty.kind() else { + bug!(); + }; + if first_cor != cor_ty { + return Instance { + def: ty::InstanceKind::FutureDropPollShim( + async_drop_in_place_poll, + first_cor, + cor_ty, + ), + args: proxy_args, + }; + } else { + return Instance { + def: ty::InstanceKind::AsyncDropGlue(async_drop_in_place_poll, cor_ty), + args: child_args, + }; + } + } + } +} + impl<'tcx> Instance<'tcx> { pub fn new(def_id: DefId, args: GenericArgsRef<'tcx>) -> Instance<'tcx> { assert!( @@ -763,6 +836,12 @@ impl<'tcx> Instance<'tcx> { ) } + pub fn resolve_async_drop_in_place_poll(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> { + let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlacePoll, None); + let args = tcx.mk_args(&[ty.into()]); + Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args, DUMMY_SP) + } + #[instrument(level = "debug", skip(tcx), ret)] pub fn fn_once_adapter_instance( tcx: TyCtxt<'tcx>, @@ -827,6 +906,9 @@ impl<'tcx> Instance<'tcx> { }; if tcx.is_lang_item(trait_item_id, coroutine_callable_item) { + if Some(coroutine_def_id) == tcx.lang_items().async_drop_in_place_poll_fn() { + return Some(resolve_async_drop_poll(tcx, rcvr_args.type_at(0))); + } let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() else { bug!() @@ -985,6 +1067,8 @@ fn polymorphize<'tcx>( } } ty::Coroutine(def_id, args) => { + // Templated coroutine can't be polymorphized + assert!(!ty.is_templated_coroutine(self.tcx)); let polymorphized_args = polymorphize(self.tcx, ty::InstanceKind::Item(def_id), args); if args == polymorphized_args { diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index ee70a6346d920..548ef8da936ec 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1764,11 +1764,14 @@ impl<'tcx> TyCtxt<'tcx> { | ty::InstanceKind::Virtual(..) | ty::InstanceKind::ClosureOnceShim { .. } | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::FutureDropPollShim(..) | ty::InstanceKind::DropGlue(..) | ty::InstanceKind::CloneShim(..) | ty::InstanceKind::ThreadLocalShim(..) | ty::InstanceKind::FnPtrAddrShim(..) | ty::InstanceKind::AsyncDropGlueCtorShim(..) => self.mir_shims(instance), + // async drop glue should be processed specifically, as a templated coroutine + ty::InstanceKind::AsyncDropGlue(_, _ty) => todo!(), } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index c2986a701954b..e74c888e52c99 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1809,6 +1809,13 @@ impl<'tcx> Ty<'tcx> { } } + pub fn is_templated_coroutine(self, tcx: TyCtxt<'_>) -> bool { + match self.kind() { + ty::Coroutine(def, ..) => tcx.is_templated_coroutine(*def), + _ => false, + } + } + /// Returns `true` when the outermost type cannot be further normalized, /// resolved, or instantiated. This includes all primitive types, but also /// things like ADTs and trait objects, since even if their arguments or diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 8be577a3e3ebe..93d64983fea47 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -77,7 +77,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { } fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let def_id = body.source.def_id().expect_local(); + let def_id = body.source.def_id().expect_local_or_templated(|v| tcx.is_templated_coroutine(v)); // Only do inlining into fn bodies. if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { @@ -333,6 +333,21 @@ impl<'tcx> Inliner<'tcx> { return Err("still needs substitution"); } + InstanceKind::AsyncDropGlue(_, ty) | InstanceKind::AsyncDropGlueCtorShim(_, ty) => { + return if ty.still_further_specializable() { + Err("still needs substitution") + } else { + Ok(()) + }; + } + InstanceKind::FutureDropPollShim(_, ty, ty2) => { + return if ty.still_further_specializable() || ty2.still_further_specializable() { + Err("still needs substitution") + } else { + Ok(()) + }; + } + // This cannot result in an immediate cycle since the callee MIR is a shim, which does // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we // do not need to catch this here, we can wait until the inliner decides to continue @@ -345,8 +360,7 @@ impl<'tcx> Inliner<'tcx> { | InstanceKind::DropGlue(..) | InstanceKind::CloneShim(..) | InstanceKind::ThreadLocalShim(..) - | InstanceKind::FnPtrAddrShim(..) - | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), + | InstanceKind::FnPtrAddrShim(..) => return Ok(()), } if self.tcx.is_constructor(callee_def_id) { @@ -1119,8 +1133,8 @@ fn try_instance_mir<'tcx>( tcx: TyCtxt<'tcx>, instance: InstanceKind<'tcx>, ) -> Result<&'tcx Body<'tcx>, &'static str> { - if let ty::InstanceKind::DropGlue(_, Some(ty)) - | ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) = instance + if let ty::InstanceKind::DropGlue(_, Some(ty)) | ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) = + instance && let ty::Adt(def, args) = ty.kind() { let fields = def.all_fields(); diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 9828e90de883e..4523bdbffb61f 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -96,7 +96,10 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( InstanceKind::FnPtrAddrShim(..) => { continue; } - InstanceKind::DropGlue(..) | InstanceKind::AsyncDropGlueCtorShim(..) => { + InstanceKind::DropGlue(..) + | InstanceKind::FutureDropPollShim(..) + | InstanceKind::AsyncDropGlue(..) + | InstanceKind::AsyncDropGlueCtorShim(..) => { // FIXME: A not fully instantiated drop shim can cause ICEs if one attempts to // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this // needs some more analysis. diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 8c2e1dedc1149..27bc213fcbdd5 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -127,6 +127,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance), ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty), ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty), + ty::InstanceKind::FutureDropPollShim(_def_id, _proxy_ty, _impl_ty) => { + todo!() + } + ty::InstanceKind::AsyncDropGlue(_def_id, _ty) => { + todo!() + } ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, _ty) => { bug!("AsyncDropGlueCtorShim in re-working ({:?})", instance) } diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 093697a290c00..78c3b14736cee 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -912,14 +912,16 @@ fn visit_instance_use<'tcx>( ty::InstanceKind::ThreadLocalShim(..) => { bug!("{:?} being reified", instance); } - ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => { + ty::InstanceKind::DropGlue(_, None) => { // Don't need to emit noop drop glue if we are calling directly. if !is_direct_call { output.push(create_fn_mono_item(tcx, instance, source)); } } ty::InstanceKind::DropGlue(_, Some(_)) - | ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::AsyncDropGlue(_, _) + | ty::InstanceKind::AsyncDropGlueCtorShim(_, _) | ty::InstanceKind::VTableShim(..) | ty::InstanceKind::ReifyShim(..) | ty::InstanceKind::ClosureOnceShim { .. } diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 0d295b8f280f5..2b15d233cc536 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -632,6 +632,8 @@ fn characteristic_def_id_of_mono_item<'tcx>( | ty::InstanceKind::CloneShim(..) | ty::InstanceKind::ThreadLocalShim(..) | ty::InstanceKind::FnPtrAddrShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::AsyncDropGlue(..) | ty::InstanceKind::AsyncDropGlueCtorShim(..) => return None, }; @@ -780,7 +782,9 @@ fn mono_item_visibility<'tcx>( let def_id = match instance.def { InstanceKind::Item(def_id) | InstanceKind::DropGlue(def_id, Some(_)) - | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_)) => def_id, + | InstanceKind::FutureDropPollShim(def_id, _, _) + | InstanceKind::AsyncDropGlue(def_id, _) + | InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id, // We match the visibility of statics here InstanceKind::ThreadLocalShim(def_id) => { @@ -796,7 +800,6 @@ fn mono_item_visibility<'tcx>( | InstanceKind::ClosureOnceShim { .. } | InstanceKind::ConstructCoroutineInClosureShim { .. } | InstanceKind::DropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) | InstanceKind::CloneShim(..) | InstanceKind::FnPtrAddrShim(..) => return Visibility::Hidden, }; diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs index a4ede374a19dc..53dc4f05586c6 100644 --- a/compiler/rustc_smir/src/rustc_smir/context.rs +++ b/compiler/rustc_smir/src/rustc_smir/context.rs @@ -601,12 +601,6 @@ impl<'tcx> Context for TablesWrapper<'tcx> { matches!(instance.def, ty::InstanceKind::DropGlue(_, None)) } - fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool { - let tables = self.0.borrow_mut(); - let instance = tables.instances[def]; - matches!(instance.def, ty::InstanceKind::AsyncDropGlueCtorShim(_, None)) - } - fn mono_instance(&self, def_id: stable_mir::DefId) -> stable_mir::mir::mono::Instance { let mut tables = self.0.borrow_mut(); let def_id = tables[def_id]; diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs index efbb0f244fc8f..52bde6520c7d9 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs @@ -855,6 +855,8 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> { | ty::InstanceKind::DropGlue(..) | ty::InstanceKind::CloneShim(..) | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::AsyncDropGlue(..) | ty::InstanceKind::AsyncDropGlueCtorShim(..) => { stable_mir::mir::mono::InstanceKind::Shim } diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs index a45b676acae73..483e649aff551 100644 --- a/compiler/rustc_span/src/def_id.rs +++ b/compiler/rustc_span/src/def_id.rs @@ -298,6 +298,20 @@ impl DefId { } } + // For templated coroutine we have non-local async_drop_in_place::coroutine def_id + #[inline] + #[track_caller] + pub fn expect_local_or_templated(self, is_templated: F) -> DefId + where + F: FnOnce(DefId) -> bool, + { + assert!( + self.is_local() || is_templated(self), + "DefId::expect_local_or_templated: `{self:?}` isn't local or templated" + ); + self + } + #[inline] pub fn is_crate_root(self) -> bool { self.index == CRATE_DEF_INDEX diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index a53d683fae342..e07a354682442 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -936,6 +936,7 @@ symbols! { fundamental, fused_iterator, future, + future_drop_poll, future_output, future_trait, gdb_script_file, diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 59ccd6dff8588..b12a18aa7c579 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -59,10 +59,17 @@ pub(super) fn mangle<'tcx>( .print_def_path( def_id, if let ty::InstanceKind::DropGlue(_, _) - | ty::InstanceKind::AsyncDropGlueCtorShim(_, _) = instance.def + | ty::InstanceKind::AsyncDropGlueCtorShim(_, _) + | ty::InstanceKind::FutureDropPollShim(_, _, _) = instance.def { // Add the name of the dropped type to the symbol name &*instance.args + } else if let ty::InstanceKind::AsyncDropGlue(_, ty) = instance.def { + let ty::Coroutine(_, cor_args) = ty.kind() else { + bug!(); + }; + let drop_ty = cor_args.first().unwrap().expect_ty(); + tcx.mk_args(&[GenericArg::from(drop_ty)]) } else { &[] }, @@ -95,6 +102,10 @@ pub(super) fn mangle<'tcx>( _ => {} } + if let ty::InstanceKind::FutureDropPollShim(..) = instance.def { + let _ = printer.write_str("{{drop-shim}}"); + } + printer.path.finish(hash) } diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 75f4499352b99..aae5faaba60ab 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -56,11 +56,17 @@ pub(super) fn mangle<'tcx>( ty::InstanceKind::ConstructCoroutineInClosureShim { receiver_by_ref: false, .. } => { Some("by_ref") } - + ty::InstanceKind::FutureDropPollShim(_, _, _) => Some("drop"), _ => None, }; - if let Some(shim_kind) = shim_kind { + if let ty::InstanceKind::AsyncDropGlue(_, ty) = instance.def { + let ty::Coroutine(_, cor_args) = ty.kind() else { + bug!(); + }; + let drop_ty = cor_args.first().unwrap().expect_ty(); + cx.print_def_path(def_id, tcx.mk_args(&[GenericArg::from(drop_ty)])).unwrap() + } else if let Some(shim_kind) = shim_kind { cx.path_append_ns(|cx| cx.print_def_path(def_id, args), 'S', 0, shim_kind).unwrap() } else { cx.print_def_path(def_id, args).unwrap() diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 16cd147b7d432..646bc4da4c2fa 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -575,8 +575,10 @@ fn fn_abi_new_uncached<'tcx>( extra_args }; - let is_drop_in_place = - fn_def_id.is_some_and(|def_id| cx.tcx.is_lang_item(def_id, LangItem::DropInPlace)); + let is_drop_in_place = fn_def_id.is_some_and(|def_id| { + cx.tcx.is_lang_item(def_id, LangItem::DropInPlace) + || cx.tcx.is_lang_item(def_id, LangItem::AsyncDropInPlace) + }); let arg_of = |ty: Ty<'tcx>, arg_idx: Option| -> Result<_, &'tcx FnAbiError<'tcx>> { let span = tracing::debug_span!("arg_of"); diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 4e175f253e984..b843c0c989ab9 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -39,19 +39,25 @@ fn resolve_instance_raw<'tcx>( if ty.needs_drop(tcx, param_env) { debug!(" => nontrivial drop glue"); match *ty.kind() { + ty::Coroutine(coroutine_def_id, ..) => { + // FIXME: sync drop of coroutine with async drop (generate both versions?) + // Currently just ignored + if tcx.optimized_mir(coroutine_def_id).coroutine_drop_async().is_some() { + ty::InstanceKind::DropGlue(def_id, None) + } else { + ty::InstanceKind::DropGlue(def_id, Some(ty)) + } + } ty::Closure(..) | ty::CoroutineClosure(..) - | ty::Coroutine(..) | ty::Tuple(..) | ty::Adt(..) | ty::Dynamic(..) | ty::Array(..) - | ty::Slice(..) => {} + | ty::Slice(..) => ty::InstanceKind::DropGlue(def_id, Some(ty)), // Drop shims can only be built from ADTs. _ => return Ok(None), } - - ty::InstanceKind::DropGlue(def_id, Some(ty)) } else { debug!(" => trivial drop glue"); ty::InstanceKind::DropGlue(def_id, None) @@ -73,11 +79,14 @@ fn resolve_instance_raw<'tcx>( _ => return Ok(None), } debug!(" => nontrivial async drop glue ctor"); - ty::InstanceKind::AsyncDropGlueCtorShim(def_id, Some(ty)) + ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) } else { debug!(" => trivial async drop glue ctor"); - ty::InstanceKind::AsyncDropGlueCtorShim(def_id, None) + ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) } + } else if tcx.is_lang_item(def_id, LangItem::AsyncDropInPlacePoll) { + let ty = args.type_at(0); + ty::InstanceKind::AsyncDropGlue(def_id, ty) } else { debug!(" => free item"); // FIXME(effects): we may want to erase the effect param if that is present on this item. diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs index 5f2d9b96c73ca..b3538779b43a9 100644 --- a/compiler/stable_mir/src/compiler_interface.rs +++ b/compiler/stable_mir/src/compiler_interface.rs @@ -178,9 +178,6 @@ pub trait Context { /// Check if this is an empty DropGlue shim. fn is_empty_drop_shim(&self, def: InstanceDef) -> bool; - /// Check if this is an empty AsyncDropGlueCtor shim. - fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool; - /// Convert a non-generic crate item into an instance. /// This function will panic if the item is generic. fn mono_instance(&self, def_id: DefId) -> Instance; diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs index cd02844453112..b12daf3c60020 100644 --- a/compiler/stable_mir/src/mir/mono.rs +++ b/compiler/stable_mir/src/mir/mono.rs @@ -162,10 +162,7 @@ impl Instance { /// When generating code for a Drop terminator, users can ignore an empty drop glue. /// These shims are only needed to generate a valid Drop call done via VTable. pub fn is_empty_shim(&self) -> bool { - self.kind == InstanceKind::Shim - && with(|cx| { - cx.is_empty_drop_shim(self.def) || cx.is_empty_async_drop_ctor_shim(self.def) - }) + self.kind == InstanceKind::Shim && with(|cx| cx.is_empty_drop_shim(self.def)) } /// Try to constant evaluate the instance into a constant with the given type.