diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index d2f11d48140c9..41201f6603d96 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -364,7 +364,7 @@ fn exported_symbols_provider_local(
));
}
MonoItem::Fn(Instance {
- def: InstanceKind::AsyncDropGlueCtorShim(def_id, Some(ty)),
+ def: InstanceKind::AsyncDropGlueCtorShim(def_id, ty),
args,
}) => {
// A little sanity-check
@@ -381,6 +381,16 @@ fn exported_symbols_provider_local(
},
));
}
+ MonoItem::Fn(Instance { def: InstanceKind::AsyncDropGlue(_, ty), args: _ }) => {
+ symbols.push((
+ ExportedSymbol::AsyncDropGlue(ty),
+ SymbolExportInfo {
+ level: SymbolExportLevel::Rust,
+ kind: SymbolExportKind::Text,
+ used: false,
+ },
+ ));
+ }
_ => {
// Any other symbols don't qualify for sharing
}
@@ -404,6 +414,7 @@ fn upstream_monomorphizations_provider(
let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn();
let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn();
+ let async_drop_in_place_poll_fn_def_id = tcx.lang_items().async_drop_in_place_poll_fn();
for &cnum in cnums.iter() {
for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() {
@@ -422,8 +433,13 @@ fn upstream_monomorphizations_provider(
if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
(async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
} else {
- // `drop_in_place` in place does not exist, don't try
- // to use it.
+ continue;
+ }
+ }
+ ExportedSymbol::AsyncDropGlue(ty) => {
+ if let Some(poll_fn_def_id) = async_drop_in_place_poll_fn_def_id {
+ (poll_fn_def_id, tcx.mk_args(&[ty.into()]))
+ } else {
continue;
}
}
@@ -575,6 +591,13 @@ pub fn symbol_name_for_instance_in_crate<'tcx>(
instantiating_crate,
)
}
+ ExportedSymbol::AsyncDropGlue(ty) => {
+ rustc_symbol_mangling::symbol_name_for_instance_in_crate(
+ tcx,
+ Instance::resolve_async_drop_in_place_poll(tcx, ty),
+ instantiating_crate,
+ )
+ }
ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(),
}
}
@@ -626,6 +649,7 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
// AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the
// target's default symbol decoration scheme.
ExportedSymbol::AsyncDropGlueCtorShim(..) => None,
+ ExportedSymbol::AsyncDropGlue(..) => None,
// NoDefId always follow the target's default symbol decoration scheme.
ExportedSymbol::NoDefId(..) => None,
// ThreadLocalShim always follow the target's default symbol decoration scheme.
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs
index 568a9a3a637be..d4c558125802d 100644
--- a/compiler/rustc_const_eval/src/interpret/call.rs
+++ b/compiler/rustc_const_eval/src/interpret/call.rs
@@ -552,6 +552,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
| ty::InstanceKind::FnPtrAddrShim(..)
| ty::InstanceKind::ThreadLocalShim(..)
| ty::InstanceKind::AsyncDropGlueCtorShim(..)
+ | ty::InstanceKind::AsyncDropGlue(..)
+ | ty::InstanceKind::FutureDropPollShim(..)
| ty::InstanceKind::Item(_) => {
// We need MIR for this fn
let Some((body, instance)) = M::find_mir_or_eval_fn(
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index f4ac6156919fa..83e6a83fedf0d 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -182,6 +182,7 @@ language_item_table! {
AsyncDrop, sym::async_drop, async_drop_trait, Target::Trait, GenericRequirement::None;
AsyncDropInPlace, sym::async_drop_in_place, async_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1);
AsyncDropInPlacePoll, sym::async_drop_in_place_poll, async_drop_in_place_poll_fn, Target::Closure, GenericRequirement::Exact(1);
+ FutureDropPoll, sym::future_drop_poll, future_drop_poll_fn, Target::Fn, GenericRequirement::Exact(1);
CoerceUnsized, sym::coerce_unsized, coerce_unsized_trait, Target::Trait, GenericRequirement::Minimum(1);
DispatchFromDyn, sym::dispatch_from_dyn, dispatch_from_dyn_trait, Target::Trait, GenericRequirement::Minimum(1);
diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs
index 0bfbd39879747..0a4c028f5918e 100644
--- a/compiler/rustc_middle/src/middle/exported_symbols.rs
+++ b/compiler/rustc_middle/src/middle/exported_symbols.rs
@@ -44,6 +44,7 @@ pub enum ExportedSymbol<'tcx> {
Generic(DefId, GenericArgsRef<'tcx>),
DropGlue(Ty<'tcx>),
AsyncDropGlueCtorShim(Ty<'tcx>),
+ AsyncDropGlue(Ty<'tcx>),
ThreadLocalShim(DefId),
NoDefId(ty::SymbolName<'tcx>),
}
@@ -63,6 +64,9 @@ impl<'tcx> ExportedSymbol<'tcx> {
ExportedSymbol::AsyncDropGlueCtorShim(ty) => {
tcx.symbol_name(ty::Instance::resolve_async_drop_in_place(tcx, ty))
}
+ ExportedSymbol::AsyncDropGlue(ty) => {
+ tcx.symbol_name(ty::Instance::resolve_async_drop_in_place_poll(tcx, ty))
+ }
ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance {
def: ty::InstanceKind::ThreadLocalShim(def_id),
args: ty::GenericArgs::empty(),
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 081a23b6ff317..2862daec51639 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -205,7 +205,13 @@ pub struct CoroutineInfo<'tcx> {
/// Coroutine drop glue. This field is populated after the state transform pass.
pub coroutine_drop: Option
>,
- /// The layout of a coroutine. This field is populated after the state transform pass.
+ /// Coroutine async drop glue.
+ pub coroutine_drop_async: Option>,
+
+ /// When coroutine has sync drop, this is async proxy calling `coroutine_drop` sync impl.
+ pub coroutine_drop_proxy_async: Option>,
+
+ /// The layout of a coroutine. Produced by the state transformation.
pub coroutine_layout: Option>,
/// If this is a coroutine then record the type of source expression that caused this coroutine
@@ -225,6 +231,8 @@ impl<'tcx> CoroutineInfo<'tcx> {
yield_ty: Some(yield_ty),
resume_ty: Some(resume_ty),
coroutine_drop: None,
+ coroutine_drop_async: None,
+ coroutine_drop_proxy_async: None,
coroutine_layout: None,
}
}
@@ -590,6 +598,26 @@ impl<'tcx> Body<'tcx> {
self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref())
}
+ #[inline]
+ pub fn coroutine_drop_async(&self) -> Option<&Body<'tcx>> {
+ self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop_async.as_ref())
+ }
+
+ #[inline]
+ pub fn coroutine_requires_async_drop(&self) -> bool {
+ self.coroutine_drop_async().is_some()
+ }
+
+ #[inline]
+ pub fn future_drop_poll(&self) -> Option<&Body<'tcx>> {
+ self.coroutine.as_ref().and_then(|coroutine| {
+ coroutine
+ .coroutine_drop_async
+ .as_ref()
+ .or(coroutine.coroutine_drop_proxy_async.as_ref())
+ })
+ }
+
#[inline]
pub fn coroutine_kind(&self) -> Option {
self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind)
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 69b5e0a79c7fa..c7cbeaf959752 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -71,6 +71,8 @@ impl<'tcx> MonoItem<'tcx> {
// statements, plus one for the terminator.
InstanceKind::Item(..)
| InstanceKind::DropGlue(..)
+ | InstanceKind::FutureDropPollShim(..)
+ | InstanceKind::AsyncDropGlue(..)
| InstanceKind::AsyncDropGlueCtorShim(..) => {
let mir = tcx.instance_mir(instance.def);
mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum()
@@ -419,6 +421,8 @@ impl<'tcx> CodegenUnit<'tcx> {
| InstanceKind::CloneShim(..)
| InstanceKind::ThreadLocalShim(..)
| InstanceKind::FnPtrAddrShim(..)
+ | InstanceKind::AsyncDropGlue(..)
+ | InstanceKind::FutureDropPollShim(..)
| InstanceKind::AsyncDropGlueCtorShim(..) => None,
}
}
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index cf1cb9a776760..90987cca409dc 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -232,9 +232,20 @@ fn dump_path<'tcx>(
}));
s
}
- ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => {
- // Unfortunately, pretty-printed typed are not very filename-friendly.
- // We dome some filtering.
+ ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => {
+ let mut s = ".".to_owned();
+ s.extend(ty.to_string().chars().filter_map(|c| match c {
+ ' ' => None,
+ ':' | '<' | '>' => Some('_'),
+ c => Some(c),
+ }));
+ s
+ }
+ ty::InstanceKind::AsyncDropGlue(_, ty) => {
+ let ty::Coroutine(_, args) = ty.kind() else {
+ bug!();
+ };
+ let ty = args.first().unwrap().expect_ty();
let mut s = ".".to_owned();
s.extend(ty.to_string().chars().filter_map(|c| match c {
' ' => None,
@@ -243,6 +254,21 @@ fn dump_path<'tcx>(
}));
s
}
+ ty::InstanceKind::FutureDropPollShim(_, proxy_cor, impl_cor) => {
+ let mut s = ".".to_owned();
+ s.extend(proxy_cor.to_string().chars().filter_map(|c| match c {
+ ' ' => None,
+ ':' | '<' | '>' => Some('_'),
+ c => Some(c),
+ }));
+ s.push_str(".");
+ s.extend(impl_cor.to_string().chars().filter_map(|c| match c {
+ ' ' => None,
+ ':' | '<' | '>' => Some('_'),
+ c => Some(c),
+ }));
+ s
+ }
_ => String::new(),
};
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 7861ac9296055..4956097191c49 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -349,17 +349,21 @@ macro_rules! make_mir_visitor {
coroutine_closure_def_id: _def_id,
receiver_by_ref: _,
} |
- ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) |
ty::InstanceKind::DropGlue(_def_id, None) => {}
ty::InstanceKind::FnPtrShim(_def_id, ty) |
ty::InstanceKind::DropGlue(_def_id, Some(ty)) |
ty::InstanceKind::CloneShim(_def_id, ty) |
ty::InstanceKind::FnPtrAddrShim(_def_id, ty) |
- ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, Some(ty)) => {
+ ty::InstanceKind::AsyncDropGlue(_def_id, ty) |
+ ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, ty) => {
// FIXME(eddyb) use a better `TyContext` here.
self.visit_ty($(& $mutability)? *ty, TyContext::Location(location));
}
+ ty::InstanceKind::FutureDropPollShim(_def_id, proxy_ty, impl_ty) => {
+ self.visit_ty($(& $mutability)? *proxy_ty, TyContext::Location(location));
+ self.visit_ty($(& $mutability)? *impl_ty, TyContext::Location(location));
+ }
}
self.visit_args(callee_args, location);
}
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index f81f50340ee00..a516f00e292da 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -1562,6 +1562,10 @@ impl<'tcx> TyCtxt<'tcx> {
self.coroutine_kind(def_id).is_some()
}
+ pub fn is_templated_coroutine(self, def_id: DefId) -> bool {
+ Some(def_id) == self.lang_items().async_drop_in_place_poll_fn()
+ }
+
/// Returns the movability of the coroutine of `def_id`, or panics
/// if given a `def_id` that is not a coroutine.
pub fn coroutine_movability(self, def_id: DefId) -> hir::Movability {
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 072951d131917..07218f1668bba 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -146,6 +146,9 @@ pub enum InstanceKind<'tcx> {
/// native support.
ThreadLocalShim(DefId),
+ /// Proxy shim for async drop of future (def_id, proxy_cor_ty, impl_cor_ty)
+ FutureDropPollShim(DefId, Ty<'tcx>, Ty<'tcx>),
+
/// `core::ptr::drop_in_place::`.
///
/// The `DefId` is for `core::ptr::drop_in_place`.
@@ -172,7 +175,13 @@ pub enum InstanceKind<'tcx> {
///
/// The `DefId` is for `core::future::async_drop::async_drop_in_place`, the `Ty`
/// is the type `T`.
- AsyncDropGlueCtorShim(DefId, Option>),
+ AsyncDropGlueCtorShim(DefId, Ty<'tcx>),
+
+ /// `core::future::async_drop::async_drop_in_place::<'_, T>::{closure}`.
+ ///
+ /// async_drop_in_place poll function implementation (for generated coroutine).
+ /// `Ty` here is `async_drop_in_place::{closure}` coroutine type, not just `T`
+ AsyncDropGlue(DefId, Ty<'tcx>),
}
impl<'tcx> Instance<'tcx> {
@@ -216,7 +225,9 @@ impl<'tcx> Instance<'tcx> {
.upstream_monomorphizations_for(def)
.and_then(|monos| monos.get(&self.args).cloned()),
InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args),
- InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => {
+ InstanceKind::AsyncDropGlue(_, _) => None,
+ InstanceKind::FutureDropPollShim(_, _, _) => None,
+ InstanceKind::AsyncDropGlueCtorShim(_, _) => {
tcx.upstream_async_drop_glue_for(self.args)
}
_ => None,
@@ -243,6 +254,8 @@ impl<'tcx> InstanceKind<'tcx> {
| InstanceKind::DropGlue(def_id, _)
| InstanceKind::CloneShim(def_id, _)
| InstanceKind::FnPtrAddrShim(def_id, _)
+ | InstanceKind::FutureDropPollShim(def_id, _, _)
+ | InstanceKind::AsyncDropGlue(def_id, _)
| InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id,
}
}
@@ -252,7 +265,9 @@ impl<'tcx> InstanceKind<'tcx> {
match self {
ty::InstanceKind::Item(def) => Some(def),
ty::InstanceKind::DropGlue(def_id, Some(_))
- | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_))
+ | InstanceKind::AsyncDropGlueCtorShim(def_id, _)
+ | InstanceKind::AsyncDropGlue(def_id, _)
+ | InstanceKind::FutureDropPollShim(def_id, ..)
| InstanceKind::ThreadLocalShim(def_id) => Some(def_id),
InstanceKind::VTableShim(..)
| InstanceKind::ReifyShim(..)
@@ -262,7 +277,6 @@ impl<'tcx> InstanceKind<'tcx> {
| InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::DropGlue(..)
- | InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::CloneShim(..)
| InstanceKind::FnPtrAddrShim(..) => None,
}
@@ -287,7 +301,9 @@ impl<'tcx> InstanceKind<'tcx> {
let def_id = match *self {
ty::InstanceKind::Item(def) => def,
ty::InstanceKind::DropGlue(_, Some(_)) => return false,
- ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => return false,
+ ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => return ty.is_coroutine(),
+ ty::InstanceKind::FutureDropPollShim(_, _, _) => return false,
+ ty::InstanceKind::AsyncDropGlue(_, _) => return false,
ty::InstanceKind::ThreadLocalShim(_) => return false,
_ => return true,
};
@@ -309,7 +325,7 @@ impl<'tcx> InstanceKind<'tcx> {
return true;
}
if let ty::InstanceKind::DropGlue(.., Some(ty))
- | ty::InstanceKind::AsyncDropGlueCtorShim(.., Some(ty)) = *self
+ | ty::InstanceKind::AsyncDropGlueCtorShim(.., ty) = *self
{
// Drop glue generally wants to be instantiated at every codegen
// unit, but without an #[inline] hint. We should make this
@@ -364,11 +380,12 @@ impl<'tcx> InstanceKind<'tcx> {
| InstanceKind::FnPtrAddrShim(..)
| InstanceKind::FnPtrShim(..)
| InstanceKind::DropGlue(_, Some(_))
- | InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => false,
+ | InstanceKind::FutureDropPollShim(..)
+ | InstanceKind::AsyncDropGlue(_, _) => false,
+ InstanceKind::AsyncDropGlueCtorShim(_, _) => false,
InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::DropGlue(..)
- | InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::Item(_)
| InstanceKind::Intrinsic(..)
| InstanceKind::ReifyShim(..)
@@ -445,8 +462,11 @@ pub fn fmt_instance(
InstanceKind::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"),
InstanceKind::CloneShim(_, ty) => write!(f, " - shim({ty})"),
InstanceKind::FnPtrAddrShim(_, ty) => write!(f, " - shim({ty})"),
- InstanceKind::AsyncDropGlueCtorShim(_, None) => write!(f, " - shim(None)"),
- InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => write!(f, " - shim(Some({ty}))"),
+ InstanceKind::FutureDropPollShim(_, proxy_ty, impl_ty) => {
+ write!(f, " - dropshim({proxy_ty}-{impl_ty})")
+ }
+ InstanceKind::AsyncDropGlue(_, ty) => write!(f, " - shim({ty})"),
+ InstanceKind::AsyncDropGlueCtorShim(_, ty) => write!(f, " - shim(Some({ty}))"),
}
}
@@ -464,6 +484,59 @@ impl<'tcx> fmt::Display for Instance<'tcx> {
}
}
+// async_drop_in_place::coroutine.poll, when T is a standart coroutine,
+// should be resolved to this coroutine's future_drop_poll (through FutureDropPollShim proxy).
+// async_drop_in_place::coroutine>::coroutine.poll,
+// when T is a standart coroutine, should be resolved to this coroutine's future_drop_poll.
+// async_drop_in_place::coroutine>::coroutine.poll,
+// when T is not a coroutine, should be resolved to the innermost
+// async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy)
+fn resolve_async_drop_poll<'tcx>(tcx: TyCtxt<'tcx>, mut cor_ty: Ty<'tcx>) -> Instance<'tcx> {
+ let first_cor = cor_ty;
+ let ty::Coroutine(_, proxy_args) = first_cor.kind() else {
+ bug!();
+ };
+ let async_drop_in_place_poll = tcx.lang_items().async_drop_in_place_poll_fn().unwrap();
+ let mut child_ty = cor_ty;
+ loop {
+ if let ty::Coroutine(child_def, child_args) = child_ty.kind() {
+ cor_ty = child_ty;
+ if *child_def == async_drop_in_place_poll {
+ child_ty = child_args.first().unwrap().expect_ty();
+ continue;
+ } else {
+ return Instance {
+ def: ty::InstanceKind::FutureDropPollShim(
+ async_drop_in_place_poll,
+ first_cor,
+ cor_ty,
+ ),
+ args: proxy_args,
+ };
+ }
+ } else {
+ let ty::Coroutine(_, child_args) = cor_ty.kind() else {
+ bug!();
+ };
+ if first_cor != cor_ty {
+ return Instance {
+ def: ty::InstanceKind::FutureDropPollShim(
+ async_drop_in_place_poll,
+ first_cor,
+ cor_ty,
+ ),
+ args: proxy_args,
+ };
+ } else {
+ return Instance {
+ def: ty::InstanceKind::AsyncDropGlue(async_drop_in_place_poll, cor_ty),
+ args: child_args,
+ };
+ }
+ }
+ }
+}
+
impl<'tcx> Instance<'tcx> {
pub fn new(def_id: DefId, args: GenericArgsRef<'tcx>) -> Instance<'tcx> {
assert!(
@@ -763,6 +836,12 @@ impl<'tcx> Instance<'tcx> {
)
}
+ pub fn resolve_async_drop_in_place_poll(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> {
+ let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlacePoll, None);
+ let args = tcx.mk_args(&[ty.into()]);
+ Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args, DUMMY_SP)
+ }
+
#[instrument(level = "debug", skip(tcx), ret)]
pub fn fn_once_adapter_instance(
tcx: TyCtxt<'tcx>,
@@ -827,6 +906,9 @@ impl<'tcx> Instance<'tcx> {
};
if tcx.is_lang_item(trait_item_id, coroutine_callable_item) {
+ if Some(coroutine_def_id) == tcx.lang_items().async_drop_in_place_poll_fn() {
+ return Some(resolve_async_drop_poll(tcx, rcvr_args.type_at(0)));
+ }
let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
else {
bug!()
@@ -985,6 +1067,8 @@ fn polymorphize<'tcx>(
}
}
ty::Coroutine(def_id, args) => {
+ // Templated coroutine can't be polymorphized
+ assert!(!ty.is_templated_coroutine(self.tcx));
let polymorphized_args =
polymorphize(self.tcx, ty::InstanceKind::Item(def_id), args);
if args == polymorphized_args {
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index ee70a6346d920..548ef8da936ec 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -1764,11 +1764,14 @@ impl<'tcx> TyCtxt<'tcx> {
| ty::InstanceKind::Virtual(..)
| ty::InstanceKind::ClosureOnceShim { .. }
| ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
+ | ty::InstanceKind::FutureDropPollShim(..)
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::CloneShim(..)
| ty::InstanceKind::ThreadLocalShim(..)
| ty::InstanceKind::FnPtrAddrShim(..)
| ty::InstanceKind::AsyncDropGlueCtorShim(..) => self.mir_shims(instance),
+ // async drop glue should be processed specifically, as a templated coroutine
+ ty::InstanceKind::AsyncDropGlue(_, _ty) => todo!(),
}
}
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index c2986a701954b..e74c888e52c99 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -1809,6 +1809,13 @@ impl<'tcx> Ty<'tcx> {
}
}
+ pub fn is_templated_coroutine(self, tcx: TyCtxt<'_>) -> bool {
+ match self.kind() {
+ ty::Coroutine(def, ..) => tcx.is_templated_coroutine(*def),
+ _ => false,
+ }
+ }
+
/// Returns `true` when the outermost type cannot be further normalized,
/// resolved, or instantiated. This includes all primitive types, but also
/// things like ADTs and trait objects, since even if their arguments or
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 8be577a3e3ebe..93d64983fea47 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -77,7 +77,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
}
fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
- let def_id = body.source.def_id().expect_local();
+ let def_id = body.source.def_id().expect_local_or_templated(|v| tcx.is_templated_coroutine(v));
// Only do inlining into fn bodies.
if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() {
@@ -333,6 +333,21 @@ impl<'tcx> Inliner<'tcx> {
return Err("still needs substitution");
}
+ InstanceKind::AsyncDropGlue(_, ty) | InstanceKind::AsyncDropGlueCtorShim(_, ty) => {
+ return if ty.still_further_specializable() {
+ Err("still needs substitution")
+ } else {
+ Ok(())
+ };
+ }
+ InstanceKind::FutureDropPollShim(_, ty, ty2) => {
+ return if ty.still_further_specializable() || ty2.still_further_specializable() {
+ Err("still needs substitution")
+ } else {
+ Ok(())
+ };
+ }
+
// This cannot result in an immediate cycle since the callee MIR is a shim, which does
// not get any optimizations run on it. Any subsequent inlining may cause cycles, but we
// do not need to catch this here, we can wait until the inliner decides to continue
@@ -345,8 +360,7 @@ impl<'tcx> Inliner<'tcx> {
| InstanceKind::DropGlue(..)
| InstanceKind::CloneShim(..)
| InstanceKind::ThreadLocalShim(..)
- | InstanceKind::FnPtrAddrShim(..)
- | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()),
+ | InstanceKind::FnPtrAddrShim(..) => return Ok(()),
}
if self.tcx.is_constructor(callee_def_id) {
@@ -1119,8 +1133,8 @@ fn try_instance_mir<'tcx>(
tcx: TyCtxt<'tcx>,
instance: InstanceKind<'tcx>,
) -> Result<&'tcx Body<'tcx>, &'static str> {
- if let ty::InstanceKind::DropGlue(_, Some(ty))
- | ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) = instance
+ if let ty::InstanceKind::DropGlue(_, Some(ty)) | ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) =
+ instance
&& let ty::Adt(def, args) = ty.kind()
{
let fields = def.all_fields();
diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs
index 9828e90de883e..4523bdbffb61f 100644
--- a/compiler/rustc_mir_transform/src/inline/cycle.rs
+++ b/compiler/rustc_mir_transform/src/inline/cycle.rs
@@ -96,7 +96,10 @@ pub(crate) fn mir_callgraph_reachable<'tcx>(
InstanceKind::FnPtrAddrShim(..) => {
continue;
}
- InstanceKind::DropGlue(..) | InstanceKind::AsyncDropGlueCtorShim(..) => {
+ InstanceKind::DropGlue(..)
+ | InstanceKind::FutureDropPollShim(..)
+ | InstanceKind::AsyncDropGlue(..)
+ | InstanceKind::AsyncDropGlueCtorShim(..) => {
// FIXME: A not fully instantiated drop shim can cause ICEs if one attempts to
// have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this
// needs some more analysis.
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 8c2e1dedc1149..27bc213fcbdd5 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -127,6 +127,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
+ ty::InstanceKind::FutureDropPollShim(_def_id, _proxy_ty, _impl_ty) => {
+ todo!()
+ }
+ ty::InstanceKind::AsyncDropGlue(_def_id, _ty) => {
+ todo!()
+ }
ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, _ty) => {
bug!("AsyncDropGlueCtorShim in re-working ({:?})", instance)
}
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 093697a290c00..78c3b14736cee 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -912,14 +912,16 @@ fn visit_instance_use<'tcx>(
ty::InstanceKind::ThreadLocalShim(..) => {
bug!("{:?} being reified", instance);
}
- ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => {
+ ty::InstanceKind::DropGlue(_, None) => {
// Don't need to emit noop drop glue if we are calling directly.
if !is_direct_call {
output.push(create_fn_mono_item(tcx, instance, source));
}
}
ty::InstanceKind::DropGlue(_, Some(_))
- | ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(_))
+ | ty::InstanceKind::FutureDropPollShim(..)
+ | ty::InstanceKind::AsyncDropGlue(_, _)
+ | ty::InstanceKind::AsyncDropGlueCtorShim(_, _)
| ty::InstanceKind::VTableShim(..)
| ty::InstanceKind::ReifyShim(..)
| ty::InstanceKind::ClosureOnceShim { .. }
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index 0d295b8f280f5..2b15d233cc536 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -632,6 +632,8 @@ fn characteristic_def_id_of_mono_item<'tcx>(
| ty::InstanceKind::CloneShim(..)
| ty::InstanceKind::ThreadLocalShim(..)
| ty::InstanceKind::FnPtrAddrShim(..)
+ | ty::InstanceKind::FutureDropPollShim(..)
+ | ty::InstanceKind::AsyncDropGlue(..)
| ty::InstanceKind::AsyncDropGlueCtorShim(..) => return None,
};
@@ -780,7 +782,9 @@ fn mono_item_visibility<'tcx>(
let def_id = match instance.def {
InstanceKind::Item(def_id)
| InstanceKind::DropGlue(def_id, Some(_))
- | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_)) => def_id,
+ | InstanceKind::FutureDropPollShim(def_id, _, _)
+ | InstanceKind::AsyncDropGlue(def_id, _)
+ | InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id,
// We match the visibility of statics here
InstanceKind::ThreadLocalShim(def_id) => {
@@ -796,7 +800,6 @@ fn mono_item_visibility<'tcx>(
| InstanceKind::ClosureOnceShim { .. }
| InstanceKind::ConstructCoroutineInClosureShim { .. }
| InstanceKind::DropGlue(..)
- | InstanceKind::AsyncDropGlueCtorShim(..)
| InstanceKind::CloneShim(..)
| InstanceKind::FnPtrAddrShim(..) => return Visibility::Hidden,
};
diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs
index a4ede374a19dc..53dc4f05586c6 100644
--- a/compiler/rustc_smir/src/rustc_smir/context.rs
+++ b/compiler/rustc_smir/src/rustc_smir/context.rs
@@ -601,12 +601,6 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
matches!(instance.def, ty::InstanceKind::DropGlue(_, None))
}
- fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool {
- let tables = self.0.borrow_mut();
- let instance = tables.instances[def];
- matches!(instance.def, ty::InstanceKind::AsyncDropGlueCtorShim(_, None))
- }
-
fn mono_instance(&self, def_id: stable_mir::DefId) -> stable_mir::mir::mono::Instance {
let mut tables = self.0.borrow_mut();
let def_id = tables[def_id];
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
index efbb0f244fc8f..52bde6520c7d9 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -855,6 +855,8 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
| ty::InstanceKind::DropGlue(..)
| ty::InstanceKind::CloneShim(..)
| ty::InstanceKind::FnPtrShim(..)
+ | ty::InstanceKind::FutureDropPollShim(..)
+ | ty::InstanceKind::AsyncDropGlue(..)
| ty::InstanceKind::AsyncDropGlueCtorShim(..) => {
stable_mir::mir::mono::InstanceKind::Shim
}
diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs
index a45b676acae73..483e649aff551 100644
--- a/compiler/rustc_span/src/def_id.rs
+++ b/compiler/rustc_span/src/def_id.rs
@@ -298,6 +298,20 @@ impl DefId {
}
}
+ // For templated coroutine we have non-local async_drop_in_place::coroutine def_id
+ #[inline]
+ #[track_caller]
+ pub fn expect_local_or_templated(self, is_templated: F) -> DefId
+ where
+ F: FnOnce(DefId) -> bool,
+ {
+ assert!(
+ self.is_local() || is_templated(self),
+ "DefId::expect_local_or_templated: `{self:?}` isn't local or templated"
+ );
+ self
+ }
+
#[inline]
pub fn is_crate_root(self) -> bool {
self.index == CRATE_DEF_INDEX
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index a53d683fae342..e07a354682442 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -936,6 +936,7 @@ symbols! {
fundamental,
fused_iterator,
future,
+ future_drop_poll,
future_output,
future_trait,
gdb_script_file,
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index 59ccd6dff8588..b12a18aa7c579 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -59,10 +59,17 @@ pub(super) fn mangle<'tcx>(
.print_def_path(
def_id,
if let ty::InstanceKind::DropGlue(_, _)
- | ty::InstanceKind::AsyncDropGlueCtorShim(_, _) = instance.def
+ | ty::InstanceKind::AsyncDropGlueCtorShim(_, _)
+ | ty::InstanceKind::FutureDropPollShim(_, _, _) = instance.def
{
// Add the name of the dropped type to the symbol name
&*instance.args
+ } else if let ty::InstanceKind::AsyncDropGlue(_, ty) = instance.def {
+ let ty::Coroutine(_, cor_args) = ty.kind() else {
+ bug!();
+ };
+ let drop_ty = cor_args.first().unwrap().expect_ty();
+ tcx.mk_args(&[GenericArg::from(drop_ty)])
} else {
&[]
},
@@ -95,6 +102,10 @@ pub(super) fn mangle<'tcx>(
_ => {}
}
+ if let ty::InstanceKind::FutureDropPollShim(..) = instance.def {
+ let _ = printer.write_str("{{drop-shim}}");
+ }
+
printer.path.finish(hash)
}
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index 75f4499352b99..aae5faaba60ab 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -56,11 +56,17 @@ pub(super) fn mangle<'tcx>(
ty::InstanceKind::ConstructCoroutineInClosureShim { receiver_by_ref: false, .. } => {
Some("by_ref")
}
-
+ ty::InstanceKind::FutureDropPollShim(_, _, _) => Some("drop"),
_ => None,
};
- if let Some(shim_kind) = shim_kind {
+ if let ty::InstanceKind::AsyncDropGlue(_, ty) = instance.def {
+ let ty::Coroutine(_, cor_args) = ty.kind() else {
+ bug!();
+ };
+ let drop_ty = cor_args.first().unwrap().expect_ty();
+ cx.print_def_path(def_id, tcx.mk_args(&[GenericArg::from(drop_ty)])).unwrap()
+ } else if let Some(shim_kind) = shim_kind {
cx.path_append_ns(|cx| cx.print_def_path(def_id, args), 'S', 0, shim_kind).unwrap()
} else {
cx.print_def_path(def_id, args).unwrap()
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 16cd147b7d432..646bc4da4c2fa 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -575,8 +575,10 @@ fn fn_abi_new_uncached<'tcx>(
extra_args
};
- let is_drop_in_place =
- fn_def_id.is_some_and(|def_id| cx.tcx.is_lang_item(def_id, LangItem::DropInPlace));
+ let is_drop_in_place = fn_def_id.is_some_and(|def_id| {
+ cx.tcx.is_lang_item(def_id, LangItem::DropInPlace)
+ || cx.tcx.is_lang_item(def_id, LangItem::AsyncDropInPlace)
+ });
let arg_of = |ty: Ty<'tcx>, arg_idx: Option| -> Result<_, &'tcx FnAbiError<'tcx>> {
let span = tracing::debug_span!("arg_of");
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 4e175f253e984..b843c0c989ab9 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -39,19 +39,25 @@ fn resolve_instance_raw<'tcx>(
if ty.needs_drop(tcx, param_env) {
debug!(" => nontrivial drop glue");
match *ty.kind() {
+ ty::Coroutine(coroutine_def_id, ..) => {
+ // FIXME: sync drop of coroutine with async drop (generate both versions?)
+ // Currently just ignored
+ if tcx.optimized_mir(coroutine_def_id).coroutine_drop_async().is_some() {
+ ty::InstanceKind::DropGlue(def_id, None)
+ } else {
+ ty::InstanceKind::DropGlue(def_id, Some(ty))
+ }
+ }
ty::Closure(..)
| ty::CoroutineClosure(..)
- | ty::Coroutine(..)
| ty::Tuple(..)
| ty::Adt(..)
| ty::Dynamic(..)
| ty::Array(..)
- | ty::Slice(..) => {}
+ | ty::Slice(..) => ty::InstanceKind::DropGlue(def_id, Some(ty)),
// Drop shims can only be built from ADTs.
_ => return Ok(None),
}
-
- ty::InstanceKind::DropGlue(def_id, Some(ty))
} else {
debug!(" => trivial drop glue");
ty::InstanceKind::DropGlue(def_id, None)
@@ -73,11 +79,14 @@ fn resolve_instance_raw<'tcx>(
_ => return Ok(None),
}
debug!(" => nontrivial async drop glue ctor");
- ty::InstanceKind::AsyncDropGlueCtorShim(def_id, Some(ty))
+ ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty)
} else {
debug!(" => trivial async drop glue ctor");
- ty::InstanceKind::AsyncDropGlueCtorShim(def_id, None)
+ ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty)
}
+ } else if tcx.is_lang_item(def_id, LangItem::AsyncDropInPlacePoll) {
+ let ty = args.type_at(0);
+ ty::InstanceKind::AsyncDropGlue(def_id, ty)
} else {
debug!(" => free item");
// FIXME(effects): we may want to erase the effect param if that is present on this item.
diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
index 5f2d9b96c73ca..b3538779b43a9 100644
--- a/compiler/stable_mir/src/compiler_interface.rs
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -178,9 +178,6 @@ pub trait Context {
/// Check if this is an empty DropGlue shim.
fn is_empty_drop_shim(&self, def: InstanceDef) -> bool;
- /// Check if this is an empty AsyncDropGlueCtor shim.
- fn is_empty_async_drop_ctor_shim(&self, def: InstanceDef) -> bool;
-
/// Convert a non-generic crate item into an instance.
/// This function will panic if the item is generic.
fn mono_instance(&self, def_id: DefId) -> Instance;
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index cd02844453112..b12daf3c60020 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -162,10 +162,7 @@ impl Instance {
/// When generating code for a Drop terminator, users can ignore an empty drop glue.
/// These shims are only needed to generate a valid Drop call done via VTable.
pub fn is_empty_shim(&self) -> bool {
- self.kind == InstanceKind::Shim
- && with(|cx| {
- cx.is_empty_drop_shim(self.def) || cx.is_empty_async_drop_ctor_shim(self.def)
- })
+ self.kind == InstanceKind::Shim && with(|cx| cx.is_empty_drop_shim(self.def))
}
/// Try to constant evaluate the instance into a constant with the given type.