Auto merge of #138499 - lcnr:borrowck-typeck_root, r=oli-obk

borrowck typeck children together with their root

This introduces new cycle errors, even with `feature(inline_const_pat)` removed, see the `non-structural-match-types-cycle-err.rs` test.

The new cycle error happens as the layout of `async`-blocks relies on their `optimized_mir`. As that now depends on `mir_borrowck` of its typeck parent, computing the layout of an `async`-block during MIR building, e.g. when evaluating a named `const` pattern. I think there's currently no way to have a named const pattern whose type references an async block while being allowed? cc `@oli-obk` `@RalfJung`

I cannot think of other cases where we currently rely on the MIR of a typeck children while borrowchecking their parent. The crater run came back without any breakage. My work here will prevent any future features which rely on this as we'll get locked into borrowchecking them together as I continue to work on https://github.com/rust-lang/types-team/issues/129, cc `@rust-lang/types.`

r? compiler-errors

compiler/rustc_borrowck/src/consumers.rs

@@ -15,6 +15,7 @@ pub use super::polonius::legacy::{
     RichLocation, RustcFacts,
 };
 pub use super::region_infer::RegionInferenceContext;
+use crate::{BorrowCheckRootCtxt, do_mir_borrowck};
 
 /// Options determining the output behavior of [`get_body_with_borrowck_facts`].
 ///
@@ -97,8 +98,9 @@ pub struct BodyWithBorrowckFacts<'tcx> {
 /// *   Polonius is highly unstable, so expect regular changes in its signature or other details.
 pub fn get_body_with_borrowck_facts(
     tcx: TyCtxt<'_>,
-    def: LocalDefId,
+    def_id: LocalDefId,
     options: ConsumerOptions,
 ) -> BodyWithBorrowckFacts<'_> {
-    *super::do_mir_borrowck(tcx, def, Some(options)).1.unwrap()
+    let mut root_cx = BorrowCheckRootCtxt::new(tcx, def_id);
+    *do_mir_borrowck(&mut root_cx, def_id, Some(options)).1.unwrap()
 }

compiler/rustc_borrowck/src/lib.rs

@@ -9,6 +9,7 @@
 #![feature(file_buffered)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
+#![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
 #![feature(rustdoc_internals)]
@@ -21,6 +22,7 @@ use std::cell::RefCell;
 use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref};
 
+use root_cx::BorrowCheckRootCtxt;
 use rustc_abi::FieldIdx;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
@@ -35,7 +37,9 @@ use rustc_infer::infer::{
 };
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
-use rustc_middle::ty::{self, ParamEnv, RegionVid, TyCtxt, TypingMode};
+use rustc_middle::ty::{
+    self, ParamEnv, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitable, TypingMode, fold_regions,
+};
 use rustc_middle::{bug, span_bug};
 use rustc_mir_dataflow::impls::{
     EverInitializedPlaces, MaybeInitializedPlaces, MaybeUninitializedPlaces,
@@ -45,7 +49,7 @@ use rustc_mir_dataflow::move_paths::{
 };
 use rustc_mir_dataflow::{Analysis, EntryStates, Results, ResultsVisitor, visit_results};
 use rustc_session::lint::builtin::{TAIL_EXPR_DROP_ORDER, UNUSED_MUT};
-use rustc_span::{Span, Symbol};
+use rustc_span::{ErrorGuaranteed, Span, Symbol};
 use smallvec::SmallVec;
 use tracing::{debug, instrument};
 
@@ -73,7 +77,6 @@ mod def_use;
 mod diagnostics;
 mod member_constraints;
 mod nll;
-mod opaque_types;
 mod path_utils;
 mod place_ext;
 mod places_conflict;
@@ -81,6 +84,7 @@ mod polonius;
 mod prefixes;
 mod region_infer;
 mod renumber;
+mod root_cx;
 mod session_diagnostics;
 mod type_check;
 mod universal_regions;
@@ -102,44 +106,202 @@ pub fn provide(providers: &mut Providers) {
     *providers = Providers { mir_borrowck, ..*providers };
 }
 
-fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
+/// Provider for `query mir_borrowck`. Similar to `typeck`, this must
+/// only be called for typeck roots which will then borrowck all
+/// nested bodies as well.
+fn mir_borrowck(
+    tcx: TyCtxt<'_>,
+    def: LocalDefId,
+) -> Result<&ConcreteOpaqueTypes<'_>, ErrorGuaranteed> {
+    assert!(!tcx.is_typeck_child(def.to_def_id()));
     let (input_body, _) = tcx.mir_promoted(def);
+    debug!("run query mir_borrowck: {}", tcx.def_path_str(def));
+
     let input_body: &Body<'_> = &input_body.borrow();
-    if input_body.should_skip() || input_body.tainted_by_errors.is_some() {
+    if let Some(guar) = input_body.tainted_by_errors {
-        debug!("Skipping borrowck because of injected body or tainted body");
+        debug!("Skipping borrowck because of tainted body");
-        // Let's make up a borrowck result! Fun times!
+        Err(guar)
-        let result = BorrowCheckResult {
+    } else if input_body.should_skip() {
-            concrete_opaque_types: FxIndexMap::default(),
+        debug!("Skipping borrowck because of injected body");
-            closure_requirements: None,
+        let opaque_types = ConcreteOpaqueTypes(Default::default());
-            used_mut_upvars: SmallVec::new(),
+        Ok(tcx.arena.alloc(opaque_types))
-            tainted_by_errors: input_body.tainted_by_errors,
+    } else {
-        };
+        let mut root_cx = BorrowCheckRootCtxt::new(tcx, def);
-        return tcx.arena.alloc(result);
+        let PropagatedBorrowCheckResults { closure_requirements, used_mut_upvars } =
+            do_mir_borrowck(&mut root_cx, def, None).0;
+        debug_assert!(closure_requirements.is_none());
+        debug_assert!(used_mut_upvars.is_empty());
+        root_cx.finalize()
+    }
+}
+
+/// Data propagated to the typeck parent by nested items.
+/// This should always be empty for the typeck root.
+#[derive(Debug)]
+struct PropagatedBorrowCheckResults<'tcx> {
+    closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
+    used_mut_upvars: SmallVec<[FieldIdx; 8]>,
+}
+
+/// After we borrow check a closure, we are left with various
+/// requirements that we have inferred between the free regions that
+/// appear in the closure's signature or on its field types. These
+/// requirements are then verified and proved by the closure's
+/// creating function. This struct encodes those requirements.
+///
+/// The requirements are listed as being between various `RegionVid`. The 0th
+/// region refers to `'static`; subsequent region vids refer to the free
+/// regions that appear in the closure (or coroutine's) type, in order of
+/// appearance. (This numbering is actually defined by the `UniversalRegions`
+/// struct in the NLL region checker. See for example
+/// `UniversalRegions::closure_mapping`.) Note the free regions in the
+/// closure's signature and captures are erased.
+///
+/// Example: If type check produces a closure with the closure args:
+///
+/// ```text
+/// ClosureArgs = [
+///     'a,                                         // From the parent.
+///     'b,
+///     i8,                                         // the "closure kind"
+///     for<'x> fn(&'<erased> &'x u32) -> &'x u32,  // the "closure signature"
+///     &'<erased> String,                          // some upvar
+/// ]
+/// ```
+///
+/// We would "renumber" each free region to a unique vid, as follows:
+///
+/// ```text
+/// ClosureArgs = [
+///     '1,                                         // From the parent.
+///     '2,
+///     i8,                                         // the "closure kind"
+///     for<'x> fn(&'3 &'x u32) -> &'x u32,         // the "closure signature"
+///     &'4 String,                                 // some upvar
+/// ]
+/// ```
+///
+/// Now the code might impose a requirement like `'1: '2`. When an
+/// instance of the closure is created, the corresponding free regions
+/// can be extracted from its type and constrained to have the given
+/// outlives relationship.
+#[derive(Clone, Debug)]
+pub struct ClosureRegionRequirements<'tcx> {
+    /// The number of external regions defined on the closure. In our
+    /// example above, it would be 3 -- one for `'static`, then `'1`
+    /// and `'2`. This is just used for a sanity check later on, to
+    /// make sure that the number of regions we see at the callsite
+    /// matches.
+    pub num_external_vids: usize,
+
+    /// Requirements between the various free regions defined in
+    /// indices.
+    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
+}
+
+/// Indicates an outlives-constraint between a type or between two
+/// free regions declared on the closure.
+#[derive(Copy, Clone, Debug)]
+pub struct ClosureOutlivesRequirement<'tcx> {
+    // This region or type ...
+    pub subject: ClosureOutlivesSubject<'tcx>,
+
+    // ... must outlive this one.
+    pub outlived_free_region: ty::RegionVid,
+
+    // If not, report an error here ...
+    pub blame_span: Span,
+
+    // ... due to this reason.
+    pub category: ConstraintCategory<'tcx>,
+}
+
+// Make sure this enum doesn't unintentionally grow
+#[cfg(target_pointer_width = "64")]
+rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16);
+
+/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
+/// that must outlive some region.
+#[derive(Copy, Clone, Debug)]
+pub enum ClosureOutlivesSubject<'tcx> {
+    /// Subject is a type, typically a type parameter, but could also
+    /// be a projection. Indicates a requirement like `T: 'a` being
+    /// passed to the caller, where the type here is `T`.
+    Ty(ClosureOutlivesSubjectTy<'tcx>),
+
+    /// Subject is a free region from the closure. Indicates a requirement
+    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
+    Region(ty::RegionVid),
+}
+
+/// Represents a `ty::Ty` for use in [`ClosureOutlivesSubject`].
+///
+/// This abstraction is necessary because the type may include `ReVar` regions,
+/// which is what we use internally within NLL code, and they can't be used in
+/// a query response.
+#[derive(Copy, Clone, Debug)]
+pub struct ClosureOutlivesSubjectTy<'tcx> {
+    inner: Ty<'tcx>,
+}
+// DO NOT implement `TypeVisitable` or `TypeFoldable` traits, because this
+// type is not recognized as a binder for late-bound region.
+impl<'tcx, I> !TypeVisitable<I> for ClosureOutlivesSubjectTy<'tcx> {}
+impl<'tcx, I> !TypeFoldable<I> for ClosureOutlivesSubjectTy<'tcx> {}
+
+impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
+    /// All regions of `ty` must be of kind `ReVar` and must represent
+    /// universal regions *external* to the closure.
+    pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
+        let inner = fold_regions(tcx, ty, |r, depth| match r.kind() {
+            ty::ReVar(vid) => {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(vid.index()),
+                    kind: ty::BoundRegionKind::Anon,
+                };
+                ty::Region::new_bound(tcx, depth, br)
+            }
+            _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
+        });
+
+        Self { inner }
     }
 
-    let borrowck_result = do_mir_borrowck(tcx, def, None).0;
+    pub fn instantiate(
-    debug!("mir_borrowck done");
+        self,
-
+        tcx: TyCtxt<'tcx>,
-    tcx.arena.alloc(borrowck_result)
+        mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
+    ) -> Ty<'tcx> {
+        fold_regions(tcx, self.inner, |r, depth| match r.kind() {
+            ty::ReBound(debruijn, br) => {
+                debug_assert_eq!(debruijn, depth);
+                map(ty::RegionVid::from_usize(br.var.index()))
+            }
+            _ => bug!("unexpected region {r:?}"),
+        })
+    }
 }
 
 /// Perform the actual borrow checking.
 ///
 /// Use `consumer_options: None` for the default behavior of returning
-/// [`BorrowCheckResult`] only. Otherwise, return [`BodyWithBorrowckFacts`] according
+/// [`PropagatedBorrowCheckResults`] only. Otherwise, return [`BodyWithBorrowckFacts`]
-/// to the given [`ConsumerOptions`].
+/// according to the given [`ConsumerOptions`].
-#[instrument(skip(tcx), level = "debug")]
+///
+/// For nested bodies this should only be called through `root_cx.get_or_insert_nested`.
+#[instrument(skip(root_cx), level = "debug")]
 fn do_mir_borrowck<'tcx>(
-    tcx: TyCtxt<'tcx>,
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     def: LocalDefId,
     consumer_options: Option<ConsumerOptions>,
-) -> (BorrowCheckResult<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+) -> (PropagatedBorrowCheckResults<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+    let tcx = root_cx.tcx;
     let infcx = BorrowckInferCtxt::new(tcx, def);
     let (input_body, promoted) = tcx.mir_promoted(def);
     let input_body: &Body<'_> = &input_body.borrow();
     let input_promoted: &IndexSlice<_, _> = &promoted.borrow();
     if let Some(e) = input_body.tainted_by_errors {
         infcx.set_tainted_by_errors(e);
+        root_cx.set_tainted_by_errors(e);
     }
 
     let mut local_names = IndexVec::from_elem(None, &input_body.local_decls);
@@ -185,13 +347,13 @@ fn do_mir_borrowck<'tcx>(
     // Compute non-lexical lifetimes.
     let nll::NllOutput {
         regioncx,
-        concrete_opaque_types,
         polonius_input,
         polonius_output,
         opt_closure_req,
         nll_errors,
         polonius_diagnostics,
     } = nll::compute_regions(
+        root_cx,
         &infcx,
         free_regions,
         body,
@@ -210,26 +372,19 @@ fn do_mir_borrowck<'tcx>(
     // We also have a `#[rustc_regions]` annotation that causes us to dump
     // information.
     let diags_buffer = &mut BorrowckDiagnosticsBuffer::default();
-    nll::dump_annotation(
+    nll::dump_annotation(&infcx, body, &regioncx, &opt_closure_req, diags_buffer);
-        &infcx,
-        body,
-        &regioncx,
-        &opt_closure_req,
-        &concrete_opaque_types,
-        diags_buffer,
-    );
 
     let movable_coroutine =
-        // The first argument is the coroutine type passed by value
+    // The first argument is the coroutine type passed by value
-        if let Some(local) = body.local_decls.raw.get(1)
+    if let Some(local) = body.local_decls.raw.get(1)
-        // Get the interior types and args which typeck computed
+    // Get the interior types and args which typeck computed
-        && let ty::Coroutine(def_id, _) = *local.ty.kind()
+    && let ty::Coroutine(def_id, _) = *local.ty.kind()
-        && tcx.coroutine_movability(def_id) == hir::Movability::Movable
+    && tcx.coroutine_movability(def_id) == hir::Movability::Movable
-    {
+{
-        true
+    true
-    } else {
+} else {
-        false
+    false
-    };
+};
 
     // While promoteds should mostly be correct by construction, we need to check them for
     // invalid moves to detect moving out of arrays:`struct S; fn main() { &([S][0]); }`.
@@ -240,6 +395,7 @@ fn do_mir_borrowck<'tcx>(
         // this check out of `MirBorrowckCtxt`, actually doing so is far from trivial.
         let move_data = MoveData::gather_moves(promoted_body, tcx, |_| true);
         let mut promoted_mbcx = MirBorrowckCtxt {
+            root_cx,
             infcx: &infcx,
             body: promoted_body,
             move_data: &move_data,
@@ -280,6 +436,7 @@ fn do_mir_borrowck<'tcx>(
     }
 
     let mut mbcx = MirBorrowckCtxt {
+        root_cx,
         infcx: &infcx,
         body,
         move_data: &move_data,
@@ -347,13 +504,13 @@ fn do_mir_borrowck<'tcx>(
 
     debug!("mbcx.used_mut: {:?}", mbcx.used_mut);
     mbcx.lint_unused_mut();
-    let tainted_by_errors = mbcx.emit_errors();
+    if let Some(guar) = mbcx.emit_errors() {
+        mbcx.root_cx.set_tainted_by_errors(guar);
+    }
 
-    let result = BorrowCheckResult {
+    let result = PropagatedBorrowCheckResults {
-        concrete_opaque_types: concrete_opaque_types.into_inner(),
         closure_requirements: opt_closure_req,
         used_mut_upvars: mbcx.used_mut_upvars,
-        tainted_by_errors,
     };
 
     let body_with_facts = if consumer_options.is_some() {
@@ -488,6 +645,7 @@ impl<'tcx> Deref for BorrowckInferCtxt<'tcx> {
 }
 
 struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
+    root_cx: &'a mut BorrowCheckRootCtxt<'tcx>,
     infcx: &'infcx BorrowckInferCtxt<'tcx>,
     body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
@@ -1361,11 +1519,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
                     | AggregateKind::CoroutineClosure(def_id, _)
                     | AggregateKind::Coroutine(def_id, _) => {
                         let def_id = def_id.expect_local();
-                        let BorrowCheckResult { used_mut_upvars, .. } =
+                        let used_mut_upvars = self.root_cx.used_mut_upvars(def_id);
-                            self.infcx.tcx.mir_borrowck(def_id);
                         debug!("{:?} used_mut_upvars={:?}", def_id, used_mut_upvars);
-                        for field in used_mut_upvars {
+                        // FIXME: We're cloning the `SmallVec` here to avoid borrowing `root_cx`
-                            self.propagate_closure_used_mut_upvar(&operands[*field]);
+                        // when calling `propagate_closure_used_mut_upvar`. This should ideally
+                        // be unnecessary.
+                        for field in used_mut_upvars.clone() {
+                            self.propagate_closure_used_mut_upvar(&operands[field]);
                         }
                     }
                     AggregateKind::Adt(..)

compiler/rustc_borrowck/src/nll.rs

@@ -8,10 +8,7 @@ use std::str::FromStr;
 use polonius_engine::{Algorithm, Output};
 use rustc_index::IndexSlice;
 use rustc_middle::mir::pretty::{PrettyPrintMirOptions, dump_mir_with_options};
-use rustc_middle::mir::{
+use rustc_middle::mir::{Body, PassWhere, Promoted, create_dump_file, dump_enabled, dump_mir};
-    Body, ClosureOutlivesSubject, ClosureRegionRequirements, PassWhere, Promoted, create_dump_file,
-    dump_enabled, dump_mir,
-};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_mir_dataflow::ResultsCursor;
@@ -25,7 +22,6 @@ use tracing::{debug, instrument};
 use crate::borrow_set::BorrowSet;
 use crate::consumers::ConsumerOptions;
 use crate::diagnostics::{BorrowckDiagnosticsBuffer, RegionErrors};
-use crate::opaque_types::ConcreteOpaqueTypes;
 use crate::polonius::PoloniusDiagnosticsContext;
 use crate::polonius::legacy::{
     PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
@@ -33,13 +29,15 @@ use crate::polonius::legacy::{
 use crate::region_infer::RegionInferenceContext;
 use crate::type_check::{self, MirTypeckResults};
 use crate::universal_regions::UniversalRegions;
-use crate::{BorrowckInferCtxt, polonius, renumber};
+use crate::{
+    BorrowCheckRootCtxt, BorrowckInferCtxt, ClosureOutlivesSubject, ClosureRegionRequirements,
+    polonius, renumber,
+};
 
 /// The output of `nll::compute_regions`. This includes the computed `RegionInferenceContext`, any
 /// closure requirements to propagate, and any generated errors.
 pub(crate) struct NllOutput<'tcx> {
     pub regioncx: RegionInferenceContext<'tcx>,
-    pub concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
     pub polonius_input: Option<Box<PoloniusFacts>>,
     pub polonius_output: Option<Box<PoloniusOutput>>,
     pub opt_closure_req: Option<ClosureRegionRequirements<'tcx>>,
@@ -78,6 +76,7 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
 ///
 /// This may result in errors being reported.
 pub(crate) fn compute_regions<'a, 'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
@@ -98,8 +97,6 @@ pub(crate) fn compute_regions<'a, 'tcx>(
 
     let location_map = Rc::new(DenseLocationMap::new(body));
 
-    let mut concrete_opaque_types = ConcreteOpaqueTypes::default();
-
     // Run the MIR type-checker.
     let MirTypeckResults {
         constraints,
@@ -107,6 +104,7 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         opaque_type_values,
         polonius_context,
     } = type_check::type_check(
+        root_cx,
         infcx,
         body,
         promoted,
@@ -117,7 +115,6 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         flow_inits,
         move_data,
         Rc::clone(&location_map),
-        &mut concrete_opaque_types,
     );
 
     // Create the region inference context, taking ownership of the
@@ -181,11 +178,10 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         infcx.set_tainted_by_errors(guar);
     }
 
-    regioncx.infer_opaque_types(infcx, opaque_type_values, &mut concrete_opaque_types);
+    regioncx.infer_opaque_types(root_cx, infcx, opaque_type_values);
 
     NllOutput {
         regioncx,
-        concrete_opaque_types,
         polonius_input: polonius_facts.map(Box::new),
         polonius_output,
         opt_closure_req: closure_region_requirements,
@@ -301,7 +297,6 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
-    concrete_opaque_types: &ConcreteOpaqueTypes<'tcx>,
     diagnostics_buffer: &mut BorrowckDiagnosticsBuffer<'infcx, 'tcx>,
 ) {
     let tcx = infcx.tcx;
@@ -318,7 +313,7 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
     // better.
 
     let def_span = tcx.def_span(body.source.def_id());
-    let mut err = if let Some(closure_region_requirements) = closure_region_requirements {
+    let err = if let Some(closure_region_requirements) = closure_region_requirements {
         let mut err = infcx.dcx().struct_span_note(def_span, "external requirements");
 
         regioncx.annotate(tcx, &mut err);
@@ -344,9 +339,7 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
         err
     };
 
-    if !concrete_opaque_types.is_empty() {
+    // FIXME(@lcnr): We currently don't dump the inferred hidden types here.
-        err.note(format!("Inferred opaque type values:\n{concrete_opaque_types:#?}"));
-    }
 
     diagnostics_buffer.buffer_non_error(err);
 }

compiler/rustc_borrowck/src/opaque_types.rs

@@ -1,55 +0,0 @@
-use rustc_data_structures::fx::FxIndexMap;
-use rustc_hir::def_id::LocalDefId;
-use rustc_middle::ty::{OpaqueHiddenType, Ty, TyCtxt};
-
-#[derive(Debug, Default)]
-pub(super) struct ConcreteOpaqueTypes<'tcx> {
-    concrete_opaque_types: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
-}
-
-impl<'tcx> ConcreteOpaqueTypes<'tcx> {
-    pub(super) fn is_empty(&self) -> bool {
-        self.concrete_opaque_types.is_empty()
-    }
-
-    pub(super) fn into_inner(self) -> FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>> {
-        self.concrete_opaque_types
-    }
-
-    /// Insert an opaque type into the list of opaque types defined by this function
-    /// after mapping the hidden type to the generic parameters of the opaque type
-    /// definition.
-    pub(super) fn insert(
-        &mut self,
-        tcx: TyCtxt<'tcx>,
-        def_id: LocalDefId,
-        hidden_ty: OpaqueHiddenType<'tcx>,
-    ) {
-        // Sometimes two opaque types are the same only after we remap the generic parameters
-        // back to the opaque type definition. E.g. we may have `OpaqueType<X, Y>` mapped to
-        // `(X, Y)` and `OpaqueType<Y, X>` mapped to `(Y, X)`, and those are the same, but we
-        // only know that once we convert the generic parameters to those of the opaque type.
-        if let Some(prev) = self.concrete_opaque_types.get_mut(&def_id) {
-            if prev.ty != hidden_ty.ty {
-                let (Ok(guar) | Err(guar)) =
-                    prev.build_mismatch_error(&hidden_ty, tcx).map(|d| d.emit());
-                prev.ty = Ty::new_error(tcx, guar);
-            }
-            // Pick a better span if there is one.
-            // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
-            prev.span = prev.span.substitute_dummy(hidden_ty.span);
-        } else {
-            self.concrete_opaque_types.insert(def_id, hidden_ty);
-        }
-    }
-
-    pub(super) fn extend_from_nested_body(
-        &mut self,
-        tcx: TyCtxt<'tcx>,
-        nested_body: &FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
-    ) {
-        for (&def_id, &hidden_ty) in nested_body {
-            self.insert(tcx, def_id, hidden_ty);
-        }
-    }
-}

compiler/rustc_borrowck/src/polonius/dump.rs

@@ -5,7 +5,7 @@ use rustc_index::IndexVec;
 use rustc_middle::mir::pretty::{
     PassWhere, PrettyPrintMirOptions, create_dump_file, dump_enabled, dump_mir_to_writer,
 };
-use rustc_middle::mir::{Body, ClosureRegionRequirements, Location};
+use rustc_middle::mir::{Body, Location};
 use rustc_middle::ty::{RegionVid, TyCtxt};
 use rustc_mir_dataflow::points::PointIndex;
 use rustc_session::config::MirIncludeSpans;
@@ -17,7 +17,7 @@ use crate::polonius::{
 };
 use crate::region_infer::values::LivenessValues;
 use crate::type_check::Locations;
-use crate::{BorrowckInferCtxt, RegionInferenceContext};
+use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext};
 
 /// `-Zdump-mir=polonius` dumps MIR annotated with NLL and polonius specific information.
 pub(crate) fn dump_polonius_mir<'tcx>(

compiler/rustc_borrowck/src/region_infer/mod.rs

@@ -13,9 +13,8 @@ use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound,
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
 use rustc_middle::bug;
 use rustc_middle::mir::{
-    AnnotationSource, BasicBlock, Body, ClosureOutlivesRequirement, ClosureOutlivesSubject,
+    AnnotationSource, BasicBlock, Body, ConstraintCategory, Local, Location, ReturnConstraint,
-    ClosureOutlivesSubjectTy, ClosureRegionRequirements, ConstraintCategory, Local, Location,
+    TerminatorKind,
-    ReturnConstraint, TerminatorKind,
 };
 use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable, UniverseIndex, fold_regions};
@@ -24,7 +23,6 @@ use rustc_span::hygiene::DesugaringKind;
 use rustc_span::{DUMMY_SP, Span};
 use tracing::{Level, debug, enabled, instrument, trace};
 
-use crate::BorrowckInferCtxt;
 use crate::constraints::graph::{self, NormalConstraintGraph, RegionGraph};
 use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstraintSet};
 use crate::dataflow::BorrowIndex;
@@ -37,6 +35,10 @@ use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, T
 use crate::type_check::free_region_relations::UniversalRegionRelations;
 use crate::type_check::{Locations, MirTypeckRegionConstraints};
 use crate::universal_regions::UniversalRegions;
+use crate::{
+    BorrowckInferCtxt, ClosureOutlivesRequirement, ClosureOutlivesSubject,
+    ClosureOutlivesSubjectTy, ClosureRegionRequirements,
+};
 
 mod dump_mir;
 mod graphviz;

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -10,7 +10,7 @@ use rustc_trait_selection::opaque_types::check_opaque_type_parameter_valid;
 use tracing::{debug, instrument};
 
 use super::RegionInferenceContext;
-use crate::opaque_types::ConcreteOpaqueTypes;
+use crate::BorrowCheckRootCtxt;
 use crate::session_diagnostics::LifetimeMismatchOpaqueParam;
 use crate::universal_regions::RegionClassification;
 
@@ -58,12 +58,12 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ///
     /// [rustc-dev-guide chapter]:
     /// https://rustc-dev-guide.rust-lang.org/opaque-types-region-infer-restrictions.html
-    #[instrument(level = "debug", skip(self, infcx), ret)]
+    #[instrument(level = "debug", skip(self, root_cx, infcx), ret)]
     pub(crate) fn infer_opaque_types(
         &self,
+        root_cx: &mut BorrowCheckRootCtxt<'tcx>,
         infcx: &InferCtxt<'tcx>,
         opaque_ty_decls: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
-        concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
     ) {
         let mut decls_modulo_regions: FxIndexMap<OpaqueTypeKey<'tcx>, (OpaqueTypeKey<'tcx>, Span)> =
             FxIndexMap::default();
@@ -140,11 +140,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 }
             }
 
-            concrete_opaque_types.insert(
+            root_cx.add_concrete_opaque_type(
-                infcx.tcx,
                 opaque_type_key.def_id,
-                OpaqueHiddenType { ty, span: concrete_type.span },
+                OpaqueHiddenType { span: concrete_type.span, ty },
             );
+
             // Check that all opaque types have the same region parameters if they have the same
             // non-region parameters. This is necessary because within the new solver we perform
             // various query operations modulo regions, and thus could unsoundly select some impls

compiler/rustc_borrowck/src/root_cx.rs

@@ -0,0 +1,101 @@
+use rustc_abi::FieldIdx;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::bug;
+use rustc_middle::ty::{OpaqueHiddenType, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::ErrorGuaranteed;
+use smallvec::SmallVec;
+
+use crate::{ClosureRegionRequirements, ConcreteOpaqueTypes, PropagatedBorrowCheckResults};
+
+/// The shared context used by both the root as well as all its nested
+/// items.
+pub(super) struct BorrowCheckRootCtxt<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    root_def_id: LocalDefId,
+    concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
+    nested_bodies: FxHashMap<LocalDefId, PropagatedBorrowCheckResults<'tcx>>,
+    tainted_by_errors: Option<ErrorGuaranteed>,
+}
+
+impl<'tcx> BorrowCheckRootCtxt<'tcx> {
+    pub(super) fn new(tcx: TyCtxt<'tcx>, root_def_id: LocalDefId) -> BorrowCheckRootCtxt<'tcx> {
+        BorrowCheckRootCtxt {
+            tcx,
+            root_def_id,
+            concrete_opaque_types: Default::default(),
+            nested_bodies: Default::default(),
+            tainted_by_errors: None,
+        }
+    }
+
+    /// Collect all defining uses of opaque types inside of this typeck root. This
+    /// expects the hidden type to be mapped to the definition parameters of the opaque
+    /// and errors if we end up with distinct hidden types.
+    pub(super) fn add_concrete_opaque_type(
+        &mut self,
+        def_id: LocalDefId,
+        hidden_ty: OpaqueHiddenType<'tcx>,
+    ) {
+        // Sometimes two opaque types are the same only after we remap the generic parameters
+        // back to the opaque type definition. E.g. we may have `OpaqueType<X, Y>` mapped to
+        // `(X, Y)` and `OpaqueType<Y, X>` mapped to `(Y, X)`, and those are the same, but we
+        // only know that once we convert the generic parameters to those of the opaque type.
+        if let Some(prev) = self.concrete_opaque_types.0.get_mut(&def_id) {
+            if prev.ty != hidden_ty.ty {
+                let guar = hidden_ty.ty.error_reported().err().unwrap_or_else(|| {
+                    let (Ok(e) | Err(e)) =
+                        prev.build_mismatch_error(&hidden_ty, self.tcx).map(|d| d.emit());
+                    e
+                });
+                prev.ty = Ty::new_error(self.tcx, guar);
+            }
+            // Pick a better span if there is one.
+            // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
+            prev.span = prev.span.substitute_dummy(hidden_ty.span);
+        } else {
+            self.concrete_opaque_types.0.insert(def_id, hidden_ty);
+        }
+    }
+
+    pub(super) fn set_tainted_by_errors(&mut self, guar: ErrorGuaranteed) {
+        self.tainted_by_errors = Some(guar);
+    }
+
+    fn get_or_insert_nested(&mut self, def_id: LocalDefId) -> &PropagatedBorrowCheckResults<'tcx> {
+        debug_assert_eq!(
+            self.tcx.typeck_root_def_id(def_id.to_def_id()),
+            self.root_def_id.to_def_id()
+        );
+        if !self.nested_bodies.contains_key(&def_id) {
+            let result = super::do_mir_borrowck(self, def_id, None).0;
+            if let Some(prev) = self.nested_bodies.insert(def_id, result) {
+                bug!("unexpected previous nested body: {prev:?}");
+            }
+        }
+
+        self.nested_bodies.get(&def_id).unwrap()
+    }
+
+    pub(super) fn closure_requirements(
+        &mut self,
+        nested_body_def_id: LocalDefId,
+    ) -> &Option<ClosureRegionRequirements<'tcx>> {
+        &self.get_or_insert_nested(nested_body_def_id).closure_requirements
+    }
+
+    pub(super) fn used_mut_upvars(
+        &mut self,
+        nested_body_def_id: LocalDefId,
+    ) -> &SmallVec<[FieldIdx; 8]> {
+        &self.get_or_insert_nested(nested_body_def_id).used_mut_upvars
+    }
+
+    pub(super) fn finalize(self) -> Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
+        if let Some(guar) = self.tainted_by_errors {
+            Err(guar)
+        } else {
+            Ok(self.tcx.arena.alloc(self.concrete_opaque_types))
+        }
+    }
+}

compiler/rustc_borrowck/src/type_check/constraint_conversion.rs

@@ -6,7 +6,6 @@ use rustc_infer::infer::region_constraints::{GenericKind, VerifyBound};
 use rustc_infer::infer::{self, InferCtxt, SubregionOrigin};
 use rustc_infer::traits::query::type_op::DeeplyNormalize;
 use rustc_middle::bug;
-use rustc_middle::mir::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory};
 use rustc_middle::ty::{
     self, GenericArgKind, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, fold_regions,
 };
@@ -18,6 +17,7 @@ use crate::constraints::OutlivesConstraint;
 use crate::region_infer::TypeTest;
 use crate::type_check::{Locations, MirTypeckRegionConstraints};
 use crate::universal_regions::UniversalRegions;
+use crate::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory};
 
 pub(crate) struct ConstraintConversion<'a, 'tcx> {
     infcx: &'a InferCtxt<'tcx>,

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -45,7 +45,6 @@ use crate::borrow_set::BorrowSet;
 use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet};
 use crate::diagnostics::UniverseInfo;
 use crate::member_constraints::MemberConstraintSet;
-use crate::opaque_types::ConcreteOpaqueTypes;
 use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable};
 use crate::polonius::{PoloniusContext, PoloniusLivenessContext};
 use crate::region_infer::TypeTest;
@@ -53,7 +52,7 @@ use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderI
 use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst};
 use crate::type_check::free_region_relations::{CreateResult, UniversalRegionRelations};
 use crate::universal_regions::{DefiningTy, UniversalRegions};
-use crate::{BorrowckInferCtxt, path_utils};
+use crate::{BorrowCheckRootCtxt, BorrowckInferCtxt, path_utils};
 
 macro_rules! span_mirbug {
     ($context:expr, $elem:expr, $($message:tt)*) => ({
@@ -102,6 +101,7 @@ mod relate_tys;
 /// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
 /// - `location_map` -- map between MIR `Location` and `PointIndex`
 pub(crate) fn type_check<'a, 'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexSlice<Promoted, Body<'tcx>>,
@@ -112,7 +112,6 @@ pub(crate) fn type_check<'a, 'tcx>(
     flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     location_map: Rc<DenseLocationMap>,
-    concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
 ) -> MirTypeckResults<'tcx> {
     let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
     let mut constraints = MirTypeckRegionConstraints {
@@ -153,6 +152,7 @@ pub(crate) fn type_check<'a, 'tcx>(
     };
 
     let mut typeck = TypeChecker {
+        root_cx,
         infcx,
         last_span: body.span,
         body,
@@ -167,7 +167,6 @@ pub(crate) fn type_check<'a, 'tcx>(
         polonius_facts,
         borrow_set,
         constraints: &mut constraints,
-        concrete_opaque_types,
         polonius_liveness,
     };
 
@@ -215,6 +214,7 @@ enum FieldAccessError {
 /// way, it accrues region constraints -- these can later be used by
 /// NLL region checking.
 struct TypeChecker<'a, 'tcx> {
+    root_cx: &'a mut BorrowCheckRootCtxt<'tcx>,
     infcx: &'a BorrowckInferCtxt<'tcx>,
     last_span: Span,
     body: &'a Body<'tcx>,
@@ -233,7 +233,6 @@ struct TypeChecker<'a, 'tcx> {
     polonius_facts: &'a mut Option<PoloniusFacts>,
     borrow_set: &'a BorrowSet<'tcx>,
     constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
-    concrete_opaque_types: &'a mut ConcreteOpaqueTypes<'tcx>,
     /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints.
     polonius_liveness: Option<PoloniusLivenessContext>,
 }
@@ -2503,11 +2502,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         args: GenericArgsRef<'tcx>,
         locations: Locations,
     ) -> ty::InstantiatedPredicates<'tcx> {
-        let closure_borrowck_results = tcx.mir_borrowck(def_id);
+        if let Some(closure_requirements) = &self.root_cx.closure_requirements(def_id) {
-        self.concrete_opaque_types
-            .extend_from_nested_body(tcx, &closure_borrowck_results.concrete_opaque_types);
-
-        if let Some(closure_requirements) = &closure_borrowck_results.closure_requirements {
             constraint_conversion::ConstraintConversion::new(
                 self.infcx,
                 self.universal_regions,

compiler/rustc_hir_analysis/src/check/check.rs

@@ -397,8 +397,11 @@ fn best_definition_site_of_opaque<'tcx>(
                 return ControlFlow::Continue(());
             }
 
-            if let Some(hidden_ty) =
+            if let Some(hidden_ty) = self
-                self.tcx.mir_borrowck(item_def_id).concrete_opaque_types.get(&self.opaque_def_id)
+                .tcx
+                .mir_borrowck(item_def_id)
+                .ok()
+                .and_then(|opaque_types| opaque_types.0.get(&self.opaque_def_id))
             {
                 ControlFlow::Break((hidden_ty.span, item_def_id))
             } else {
@@ -413,9 +416,6 @@ fn best_definition_site_of_opaque<'tcx>(
             self.tcx
         }
         fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) -> Self::Result {
-            if let hir::ExprKind::Closure(closure) = ex.kind {
-                self.check(closure.def_id)?;
-            }
             intravisit::walk_expr(self, ex)
         }
         fn visit_item(&mut self, it: &'tcx hir::Item<'tcx>) -> Self::Result {

compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs

@@ -183,25 +183,23 @@ impl<'tcx> TaitConstraintLocator<'tcx> {
                     self.non_defining_use_in_defining_scope(item_def_id);
                 }
             }
-            DefiningScopeKind::MirBorrowck => {
+            DefiningScopeKind::MirBorrowck => match tcx.mir_borrowck(item_def_id) {
-                let borrowck_result = tcx.mir_borrowck(item_def_id);
+                Err(guar) => self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar)),
-                if let Some(guar) = borrowck_result.tainted_by_errors {
+                Ok(concrete_opaque_types) => {
-                    self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
+                    if let Some(&hidden_type) = concrete_opaque_types.0.get(&self.def_id) {
-                } else if let Some(&hidden_type) =
+                        debug!(?hidden_type, "found constraint");
-                    borrowck_result.concrete_opaque_types.get(&self.def_id)
+                        self.insert_found(hidden_type);
-                {
+                    } else if let Err(guar) = tcx
-                    debug!(?hidden_type, "found constraint");
+                        .type_of_opaque_hir_typeck(self.def_id)
-                    self.insert_found(hidden_type);
+                        .instantiate_identity()
-                } else if let Err(guar) = tcx
+                        .error_reported()
-                    .type_of_opaque_hir_typeck(self.def_id)
+                    {
-                    .instantiate_identity()
+                        self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
-                    .error_reported()
+                    } else {
-                {
+                        self.non_defining_use_in_defining_scope(item_def_id);
-                    self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
+                    }
-                } else {
-                    self.non_defining_use_in_defining_scope(item_def_id);
                 }
-            }
+            },
         }
     }
 }
@@ -264,20 +262,20 @@ pub(super) fn find_opaque_ty_constraints_for_rpit<'tcx>(
                 Ty::new_diverging_default(tcx)
             }
         }
-        DefiningScopeKind::MirBorrowck => {
+        DefiningScopeKind::MirBorrowck => match tcx.mir_borrowck(owner_def_id) {
-            let borrowck_result = tcx.mir_borrowck(owner_def_id);
+            Ok(concrete_opaque_types) => {
-            if let Some(guar) = borrowck_result.tainted_by_errors {
+                if let Some(hidden_ty) = concrete_opaque_types.0.get(&def_id) {
-                Ty::new_error(tcx, guar)
+                    hidden_ty.ty
-            } else if let Some(hidden_ty) = borrowck_result.concrete_opaque_types.get(&def_id) {
-                hidden_ty.ty
-            } else {
-                let hir_ty = tcx.type_of_opaque_hir_typeck(def_id).instantiate_identity();
-                if let Err(guar) = hir_ty.error_reported() {
-                    Ty::new_error(tcx, guar)
                 } else {
-                    hir_ty
+                    let hir_ty = tcx.type_of_opaque_hir_typeck(def_id).instantiate_identity();
+                    if let Err(guar) = hir_ty.error_reported() {
+                        Ty::new_error(tcx, guar)
+                    } else {
+                        hir_ty
+                    }
                 }
             }
-        }
+            Err(guar) => Ty::new_error(tcx, guar),
+        },
     }
 }

compiler/rustc_interface/src/passes.rs

@@ -955,7 +955,9 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             // Run unsafety check because it's responsible for stealing and
             // deallocating THIR.
             tcx.ensure_ok().check_unsafety(def_id);
-            tcx.ensure_ok().mir_borrowck(def_id)
+            if !tcx.is_typeck_child(def_id.to_def_id()) {
+                tcx.ensure_ok().mir_borrowck(def_id)
+            }
         });
     });
     sess.time("MIR_effect_checking", || {

compiler/rustc_middle/src/arena.rs

@@ -28,7 +28,7 @@ macro_rules! arena_types {
                     rustc_middle::mir::Body<'tcx>
                 >,
             [decode] typeck_results: rustc_middle::ty::TypeckResults<'tcx>,
-            [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<'tcx>,
+            [decode] borrowck_result: rustc_middle::mir::ConcreteOpaqueTypes<'tcx>,
             [] resolver: rustc_data_structures::steal::Steal<(
                 rustc_middle::ty::ResolverAstLowering,
                 std::sync::Arc<rustc_ast::Crate>,

compiler/rustc_middle/src/mir/query.rs

@@ -6,14 +6,13 @@ use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::LocalDefId;
+use rustc_index::IndexVec;
 use rustc_index::bit_set::BitMatrix;
-use rustc_index::{Idx, IndexVec};
 use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 use rustc_span::{Span, Symbol};
-use smallvec::SmallVec;
 
 use super::{ConstValue, SourceInfo};
-use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty, TyCtxt, fold_regions};
+use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty};
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
@@ -85,16 +84,11 @@ impl Debug for CoroutineLayout<'_> {
     }
 }
 
-#[derive(Debug, TyEncodable, TyDecodable, HashStable)]
+/// All the opaque types that are restricted to concrete types
-pub struct BorrowCheckResult<'tcx> {
+/// by this function. Unlike the value in `TypeckResults`, this has
-    /// All the opaque types that are restricted to concrete types
+/// unerased regions.
-    /// by this function. Unlike the value in `TypeckResults`, this has
+#[derive(Default, Debug, TyEncodable, TyDecodable, HashStable)]
-    /// unerased regions.
+pub struct ConcreteOpaqueTypes<'tcx>(pub FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>);
-    pub concrete_opaque_types: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
-    pub closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
-    pub used_mut_upvars: SmallVec<[FieldIdx; 8]>,
-    pub tainted_by_errors: Option<ErrorGuaranteed>,
-}
 
 /// The result of the `mir_const_qualif` query.
 ///
@@ -108,84 +102,6 @@ pub struct ConstQualifs {
     pub needs_non_const_drop: bool,
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 }
-
-/// After we borrow check a closure, we are left with various
-/// requirements that we have inferred between the free regions that
-/// appear in the closure's signature or on its field types. These
-/// requirements are then verified and proved by the closure's
-/// creating function. This struct encodes those requirements.
-///
-/// The requirements are listed as being between various `RegionVid`. The 0th
-/// region refers to `'static`; subsequent region vids refer to the free
-/// regions that appear in the closure (or coroutine's) type, in order of
-/// appearance. (This numbering is actually defined by the `UniversalRegions`
-/// struct in the NLL region checker. See for example
-/// `UniversalRegions::closure_mapping`.) Note the free regions in the
-/// closure's signature and captures are erased.
-///
-/// Example: If type check produces a closure with the closure args:
-///
-/// ```text
-/// ClosureArgs = [
-///     'a,                                         // From the parent.
-///     'b,
-///     i8,                                         // the "closure kind"
-///     for<'x> fn(&'<erased> &'x u32) -> &'x u32,  // the "closure signature"
-///     &'<erased> String,                          // some upvar
-/// ]
-/// ```
-///
-/// We would "renumber" each free region to a unique vid, as follows:
-///
-/// ```text
-/// ClosureArgs = [
-///     '1,                                         // From the parent.
-///     '2,
-///     i8,                                         // the "closure kind"
-///     for<'x> fn(&'3 &'x u32) -> &'x u32,         // the "closure signature"
-///     &'4 String,                                 // some upvar
-/// ]
-/// ```
-///
-/// Now the code might impose a requirement like `'1: '2`. When an
-/// instance of the closure is created, the corresponding free regions
-/// can be extracted from its type and constrained to have the given
-/// outlives relationship.
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureRegionRequirements<'tcx> {
-    /// The number of external regions defined on the closure. In our
-    /// example above, it would be 3 -- one for `'static`, then `'1`
-    /// and `'2`. This is just used for a sanity check later on, to
-    /// make sure that the number of regions we see at the callsite
-    /// matches.
-    pub num_external_vids: usize,
-
-    /// Requirements between the various free regions defined in
-    /// indices.
-    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
-}
-
-/// Indicates an outlives-constraint between a type or between two
-/// free regions declared on the closure.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureOutlivesRequirement<'tcx> {
-    // This region or type ...
-    pub subject: ClosureOutlivesSubject<'tcx>,
-
-    // ... must outlive this one.
-    pub outlived_free_region: ty::RegionVid,
-
-    // If not, report an error here ...
-    pub blame_span: Span,
-
-    // ... due to this reason.
-    pub category: ConstraintCategory<'tcx>,
-}
-
-// Make sure this enum doesn't unintentionally grow
-#[cfg(target_pointer_width = "64")]
-rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16);
-
 /// Outlives-constraints can be categorized to determine whether and why they
 /// are interesting (for error reporting). Order of variants indicates sort
 /// order of the category, thereby influencing diagnostic output.
@@ -253,66 +169,6 @@ pub enum AnnotationSource {
     GenericArg,
 }
 
-/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
-/// that must outlive some region.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub enum ClosureOutlivesSubject<'tcx> {
-    /// Subject is a type, typically a type parameter, but could also
-    /// be a projection. Indicates a requirement like `T: 'a` being
-    /// passed to the caller, where the type here is `T`.
-    Ty(ClosureOutlivesSubjectTy<'tcx>),
-
-    /// Subject is a free region from the closure. Indicates a requirement
-    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
-    Region(ty::RegionVid),
-}
-
-/// Represents a `ty::Ty` for use in [`ClosureOutlivesSubject`].
-///
-/// This abstraction is necessary because the type may include `ReVar` regions,
-/// which is what we use internally within NLL code, and they can't be used in
-/// a query response.
-///
-/// DO NOT implement `TypeVisitable` or `TypeFoldable` traits, because this
-/// type is not recognized as a binder for late-bound region.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureOutlivesSubjectTy<'tcx> {
-    inner: Ty<'tcx>,
-}
-
-impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
-    /// All regions of `ty` must be of kind `ReVar` and must represent
-    /// universal regions *external* to the closure.
-    pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
-        let inner = fold_regions(tcx, ty, |r, depth| match r.kind() {
-            ty::ReVar(vid) => {
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::new(vid.index()),
-                    kind: ty::BoundRegionKind::Anon,
-                };
-                ty::Region::new_bound(tcx, depth, br)
-            }
-            _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
-        });
-
-        Self { inner }
-    }
-
-    pub fn instantiate(
-        self,
-        tcx: TyCtxt<'tcx>,
-        mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
-    ) -> Ty<'tcx> {
-        fold_regions(tcx, self.inner, |r, depth| match r.kind() {
-            ty::ReBound(debruijn, br) => {
-                debug_assert_eq!(debruijn, depth);
-                map(ty::RegionVid::new(br.var.index()))
-            }
-            _ => bug!("unexpected region {r:?}"),
-        })
-    }
-}
-
 /// The constituent parts of a mir constant of kind ADT or array.
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {

compiler/rustc_middle/src/query/mod.rs

@@ -1153,11 +1153,10 @@ rustc_queries! {
         return_result_from_ensure_ok
     }
 
-    /// Borrow-checks the function body. If this is a closure, returns
+    /// Borrow-checks the given typeck root, e.g. functions, const/static items,
-    /// additional requirements that the closure's creator must verify.
+    /// and its children, e.g. closures, inline consts.
-    query mir_borrowck(key: LocalDefId) -> &'tcx mir::BorrowCheckResult<'tcx> {
+    query mir_borrowck(key: LocalDefId) -> Result<&'tcx mir::ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
         desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key) }
-        cache_on_disk_if(tcx) { tcx.is_typeck_child(key.to_def_id()) }
     }
 
     /// Gets a complete map from all types to their inherent impls.

compiler/rustc_middle/src/ty/codec.rs

@@ -501,7 +501,7 @@ impl_decodable_via_ref! {
     &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     &'tcx traits::ImplSource<'tcx, ()>,
     &'tcx mir::Body<'tcx>,
-    &'tcx mir::BorrowCheckResult<'tcx>,
+    &'tcx mir::ConcreteOpaqueTypes<'tcx>,
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>,
     &'tcx ty::List<FieldIdx>,

compiler/rustc_mir_transform/src/lib.rs

@@ -498,8 +498,11 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     }
 
     // We only need to borrowck non-synthetic MIR.
-    let tainted_by_errors =
+    let tainted_by_errors = if !tcx.is_synthetic_mir(def) {
-        if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None };
+        tcx.mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()).err()
+    } else {
+        None
+    };
 
     let is_fn_like = tcx.def_kind(def).is_fn_like();
     if is_fn_like {
@@ -795,7 +798,7 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_
     }
 
     if !tcx.is_synthetic_mir(def) {
-        tcx.ensure_done().mir_borrowck(def);
+        tcx.ensure_done().mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
     }
     let mut promoted = tcx.mir_promoted(def).1.steal();
 

tests/ui/pattern/non-structural-match-types-cycle-err.rs

@@ -0,0 +1,24 @@
+//@ edition:2021
+
+struct AnyOption<T>(T);
+impl<T> AnyOption<T> {
+    const NONE: Option<T> = None;
+}
+
+// This is an unfortunate side-effect of borrowchecking nested items
+// together with their parent. Evaluating the `AnyOption::<_>::NONE`
+// pattern for exhaustiveness checking relies on the layout of the
+// async block. This layout relies on `optimized_mir` of the nested
+// item which is now borrowck'd together with its parent. As
+// borrowck of the parent requires us to have already lowered the match,
+// this is a query cycle.
+
+fn uwu() {}
+fn defines() {
+    match Some(async {}) {
+        AnyOption::<_>::NONE => {}
+        //~^ ERROR cycle detected when building THIR for `defines`
+        _ => {}
+    }
+}
+fn main() {}

tests/ui/pattern/non-structural-match-types-cycle-err.stderr

@@ -0,0 +1,64 @@
+error[E0391]: cycle detected when building THIR for `defines`
+  --> $DIR/non-structural-match-types-cycle-err.rs:19:9
+   |
+LL |         AnyOption::<_>::NONE => {}
+   |         ^^^^^^^^^^^^^^^^^^^^
+   |
+note: ...which requires evaluating type-level constant...
+  --> $DIR/non-structural-match-types-cycle-err.rs:5:5
+   |
+LL |     const NONE: Option<T> = None;
+   |     ^^^^^^^^^^^^^^^^^^^^^
+note: ...which requires const-evaluating + checking `<impl at $DIR/non-structural-match-types-cycle-err.rs:4:1: 4:21>::NONE`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:5:5
+   |
+LL |     const NONE: Option<T> = None;
+   |     ^^^^^^^^^^^^^^^^^^^^^
+   = note: ...which requires computing layout of `core::option::Option<{async block@$DIR/non-structural-match-types-cycle-err.rs:18:16: 18:21}>`...
+   = note: ...which requires computing layout of `{async block@$DIR/non-structural-match-types-cycle-err.rs:18:16: 18:21}`...
+note: ...which requires optimizing MIR for `defines::{closure#0}`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:18:16
+   |
+LL |     match Some(async {}) {
+   |                ^^^^^
+note: ...which requires elaborating drops for `defines::{closure#0}`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:18:16
+   |
+LL |     match Some(async {}) {
+   |                ^^^^^
+note: ...which requires borrow-checking `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires promoting constants in MIR for `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires checking if `defines` contains FFI-unwind calls...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires building MIR for `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires match-checking `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+   = note: ...which again requires building THIR for `defines`, completing the cycle
+note: cycle used when unsafety-checking `defines`
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0391`.

tests/ui/pattern/non-structural-match-types.rs

@@ -1,6 +1,4 @@
 //@ edition:2021
-#![feature(const_async_blocks)]
-
 struct AnyOption<T>(T);
 impl<T> AnyOption<T> {
     const NONE: Option<T> = None;
@@ -19,11 +17,5 @@ fn defines() {
         //~^ ERROR constant of non-structural type
         _ => {}
     }
-
-    match Some(async {}) {
-        AnyOption::<_>::NONE => {}
-        //~^ ERROR constant of non-structural type
-        _ => {}
-    }
 }
 fn main() {}

tests/ui/pattern/non-structural-match-types.stderr

@@ -1,5 +1,5 @@
 error: constant of non-structural type `Option<fn() {uwu}>` in a pattern
-  --> $DIR/non-structural-match-types.rs:12:9
+  --> $DIR/non-structural-match-types.rs:10:9
    |
 LL | impl<T> AnyOption<T> {
    | --------------------
@@ -11,8 +11,8 @@ LL |         AnyOption::<_>::NONE => {}
    |
    = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
 
-error: constant of non-structural type `Option<{closure@$DIR/non-structural-match-types.rs:17:16: 17:18}>` in a pattern
+error: constant of non-structural type `Option<{closure@$DIR/non-structural-match-types.rs:15:16: 15:18}>` in a pattern
-  --> $DIR/non-structural-match-types.rs:18:9
+  --> $DIR/non-structural-match-types.rs:16:9
    |
 LL | impl<T> AnyOption<T> {
    | --------------------
@@ -24,19 +24,5 @@ LL |         AnyOption::<_>::NONE => {}
    |
    = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
 
-error: constant of non-structural type `Option<{async block@$DIR/non-structural-match-types.rs:23:16: 23:21}>` in a pattern
+error: aborting due to 2 previous errors
-  --> $DIR/non-structural-match-types.rs:24:9
-   |
-LL | impl<T> AnyOption<T> {
-   | --------------------
-LL |     const NONE: Option<T> = None;
-   |     --------------------- constant defined here
-...
-LL |         AnyOption::<_>::NONE => {}
-   |         ^^^^^^^^^^^^^^^^^^^^ constant of non-structural type
-   |
-   = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
-   = note: `ResumeTy` must be annotated with `#[derive(PartialEq)]` to be usable in patterns
-
-error: aborting due to 3 previous errors