diff --git a/src/librustc/arena.rs b/src/librustc/arena.rs index e8c3914e695..a38dbbdd50c 100644 --- a/src/librustc/arena.rs +++ b/src/librustc/arena.rs @@ -25,6 +25,16 @@ macro_rules! arena_types { [] adt_def: rustc::ty::AdtDef, [] steal_mir: rustc::ty::steal::Steal>, [] mir: rustc::mir::Body<$tcx>, + [] steal_promoted: rustc::ty::steal::Steal< + rustc_data_structures::indexed_vec::IndexVec< + rustc::mir::Promoted, + rustc::mir::Body<$tcx> + > + >, + [] promoted: rustc_data_structures::indexed_vec::IndexVec< + rustc::mir::Promoted, + rustc::mir::Body<$tcx> + >, [] tables: rustc::ty::TypeckTables<$tcx>, [] const_allocs: rustc::mir::interpret::Allocation, [] vtable_method: Option<( diff --git a/src/librustc/mir/mod.rs b/src/librustc/mir/mod.rs index 11701a66377..66f5eaeeda1 100644 --- a/src/librustc/mir/mod.rs +++ b/src/librustc/mir/mod.rs @@ -108,11 +108,6 @@ pub struct Body<'tcx> { /// needn't) be tracked across crates. pub source_scope_local_data: ClearCrossCrate>, - /// Rvalues promoted from this function, such as borrows of constants. - /// Each of them is the Body of a constant with the fn's type parameters - /// in scope, but a separate set of locals. - pub promoted: IndexVec>, - /// Yields type of the function, if it is a generator. pub yield_ty: Option>, @@ -174,7 +169,6 @@ impl<'tcx> Body<'tcx> { basic_blocks: IndexVec>, source_scopes: IndexVec, source_scope_local_data: ClearCrossCrate>, - promoted: IndexVec>, yield_ty: Option>, local_decls: LocalDecls<'tcx>, user_type_annotations: CanonicalUserTypeAnnotations<'tcx>, @@ -196,7 +190,6 @@ impl<'tcx> Body<'tcx> { basic_blocks, source_scopes, source_scope_local_data, - promoted, yield_ty, generator_drop: None, generator_layout: None, @@ -418,7 +411,6 @@ impl_stable_hash_for!(struct Body<'tcx> { basic_blocks, source_scopes, source_scope_local_data, - promoted, yield_ty, generator_drop, generator_layout, @@ -3032,7 +3024,6 @@ BraceStructTypeFoldableImpl! { basic_blocks, source_scopes, source_scope_local_data, - promoted, yield_ty, generator_drop, generator_layout, diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index 5ab1b90642a..e1dbaeb5b17 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -110,7 +110,7 @@ rustc_queries! { no_hash } - query mir_validated(_: DefId) -> &'tcx Steal> { + query mir_validated(_: DefId) -> (&'tcx Steal>, &'tcx Steal>>) { no_hash } @@ -125,7 +125,17 @@ rustc_queries! { } } - query promoted_mir(key: DefId) -> &'tcx IndexVec> { } + query promoted_mir(key: DefId) -> &'tcx IndexVec> { + cache_on_disk_if { key.is_local() } + load_cached(tcx, id) { + let promoted: Option< + rustc_data_structures::indexed_vec::IndexVec< + crate::mir::Promoted, + crate::mir::Body<'tcx> + >> = tcx.queries.on_disk_cache.try_load_query_result(tcx, id); + promoted.map(|p| &*tcx.arena.alloc(p)) + } + } } TypeChecking { diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index e72efdb057a..9f316e93111 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -21,7 +21,7 @@ use crate::middle::cstore::EncodedMetadata; use crate::middle::lang_items; use crate::middle::resolve_lifetime::{self, ObjectLifetimeDefault}; use crate::middle::stability; -use crate::mir::{Body, interpret, ProjectionKind}; +use crate::mir::{Body, interpret, ProjectionKind, Promoted}; use crate::mir::interpret::{ConstValue, Allocation, Scalar}; use crate::ty::subst::{Kind, InternalSubsts, SubstsRef, Subst}; use crate::ty::ReprOptions; @@ -1096,6 +1096,16 @@ impl<'tcx> TyCtxt<'tcx> { self.arena.alloc(Steal::new(mir)) } + pub fn alloc_steal_promoted(self, promoted: IndexVec>) -> + &'tcx Steal>> { + self.arena.alloc(Steal::new(promoted)) + } + + pub fn intern_promoted(self, promoted: IndexVec>) -> + &'tcx IndexVec> { + self.arena.alloc(promoted) + } + pub fn alloc_adt_def( self, did: DefId, diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index a66da32fa4d..7aeeef00ea9 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -136,6 +136,15 @@ provide! { <'tcx> tcx, def_id, other, cdata, mir } + promoted_mir => { + let promoted = cdata.maybe_get_promoted_mir(tcx, def_id.index).unwrap_or_else(|| { + bug!("get_promoted_mir: missing promoted MIR for `{:?}`", def_id) + }); + + let promoted = tcx.arena.alloc(promoted); + + promoted + } mir_const_qualif => { (cdata.mir_const_qualif(def_id.index), tcx.arena.alloc(BitSet::new_empty(0))) } diff --git a/src/librustc_metadata/decoder.rs b/src/librustc_metadata/decoder.rs index da96728d2de..128e30be799 100644 --- a/src/librustc_metadata/decoder.rs +++ b/src/librustc_metadata/decoder.rs @@ -3,6 +3,7 @@ use crate::cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule, FullProcMacro}; use crate::schema::*; +use rustc_data_structures::indexed_vec::IndexVec; use rustc_data_structures::sync::{Lrc, ReadGuard}; use rustc::hir::map::{DefKey, DefPath, DefPathData, DefPathHash}; use rustc::hir; @@ -17,7 +18,7 @@ use rustc::mir::interpret::AllocDecodingSession; use rustc::session::Session; use rustc::ty::{self, Ty, TyCtxt}; use rustc::ty::codec::TyDecoder; -use rustc::mir::Body; +use rustc::mir::{Body, Promoted}; use rustc::util::captures::Captures; use std::io; @@ -923,6 +924,13 @@ impl<'a, 'tcx> CrateMetadata { } } + pub fn maybe_get_promoted_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Option>> { + match self.is_proc_macro(id) { + true => None, + false => self.entry(id).promoted_mir.map(|promoted| promoted.decode((self, tcx)),) + } + } + pub fn mir_const_qualif(&self, id: DefIndex) -> u8 { match self.entry(id).kind { EntryKind::Const(qualif, _) | diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index df3320c64a9..f3863fd788a 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -8,6 +8,7 @@ use rustc::hir::def_id::{CrateNum, CRATE_DEF_INDEX, DefIndex, DefId, LocalDefId, use rustc::hir::GenericParamKind; use rustc::hir::map::definitions::DefPathTable; use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::indexed_vec::IndexVec; use rustc::middle::dependency_format::Linkage; use rustc::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel, metadata_symbol_name}; @@ -623,6 +624,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -677,6 +679,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -713,7 +716,8 @@ impl EncodeContext<'tcx> { predicates: None, predicates_defined_on: None, - mir: None + mir: None, + promoted_mir: None, } } @@ -748,6 +752,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: None, + promoted_mir: None, } } @@ -808,6 +813,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -923,6 +929,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -1022,6 +1029,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: if mir { self.encode_optimized_mir(def_id) } else { None }, + promoted_mir: if mir { self.encode_promoted_mir(def_id) } else { None }, } } @@ -1052,6 +1060,16 @@ impl EncodeContext<'tcx> { } } + fn encode_promoted_mir(&mut self, def_id: DefId) -> Option>>> { + debug!("EncodeContext::encode_promoted_mir({:?})", def_id); + if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id) { + let promoted = self.tcx.promoted_mir(def_id); + Some(self.lazy(promoted)) + } else { + None + } + } + // Encodes the inherent implementations of a structure, enumeration, or trait. fn encode_inherent_implementations(&mut self, def_id: DefId) -> Lazy<[DefIndex]> { debug!("EncodeContext::encode_inherent_implementations({:?})", def_id); @@ -1202,6 +1220,20 @@ impl EncodeContext<'tcx> { hir::ItemKind::Use(..) => bug!("cannot encode info for item {:?}", item), }; + let mir = match item.node { + hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => true, + hir::ItemKind::Fn(_, header, ..) => { + let generics = tcx.generics_of(def_id); + let needs_inline = + (generics.requires_monomorphization(tcx) || + tcx.codegen_fn_attrs(def_id).requests_inline()) && + !self.metadata_output_only(); + let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; + needs_inline || header.constness == hir::Constness::Const || always_encode_mir + } + _ => false, + }; + Entry { kind, visibility: self.lazy(ty::Visibility::from_hir(&item.vis, item.hir_id, tcx)), @@ -1301,29 +1333,8 @@ impl EncodeContext<'tcx> { _ => None, // not *wrong* for other kinds of items, but not needed }, - mir: match item.node { - hir::ItemKind::Static(..) => { - self.encode_optimized_mir(def_id) - } - hir::ItemKind::Const(..) => self.encode_optimized_mir(def_id), - hir::ItemKind::Fn(_, header, ..) => { - let generics = tcx.generics_of(def_id); - let needs_inline = - (generics.requires_monomorphization(tcx) || - tcx.codegen_fn_attrs(def_id).requests_inline()) && - !self.metadata_output_only(); - let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; - if needs_inline - || header.constness == hir::Constness::Const - || always_encode_mir - { - self.encode_optimized_mir(def_id) - } else { - None - } - } - _ => None, - }, + mir: if mir { self.encode_optimized_mir(def_id) } else { None }, + promoted_mir: if mir { self.encode_promoted_mir(def_id) } else { None }, } } @@ -1350,6 +1361,7 @@ impl EncodeContext<'tcx> { predicates: None, predicates_defined_on: None, mir: None, + promoted_mir: None, } } @@ -1376,6 +1388,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: None, + promoted_mir: None, } } @@ -1436,6 +1449,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -1464,6 +1478,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: self.encode_optimized_mir(def_id), + promoted_mir: self.encode_promoted_mir(def_id), } } @@ -1675,6 +1690,7 @@ impl EncodeContext<'tcx> { predicates_defined_on: None, mir: None, + promoted_mir: None, } } } diff --git a/src/librustc_metadata/schema.rs b/src/librustc_metadata/schema.rs index f37877b437e..72a4b527c93 100644 --- a/src/librustc_metadata/schema.rs +++ b/src/librustc_metadata/schema.rs @@ -11,6 +11,7 @@ use rustc::session::CrateDisambiguator; use rustc::session::config::SymbolManglingVersion; use rustc::ty::{self, Ty, ReprOptions}; use rustc_target::spec::{PanicStrategy, TargetTriple}; +use rustc_data_structures::indexed_vec::IndexVec; use rustc_data_structures::svh::Svh; use syntax::{ast, attr}; @@ -231,6 +232,7 @@ pub struct Entry<'tcx> { pub predicates_defined_on: Option>>, pub mir: Option>>, + pub promoted_mir: Option>>>, } #[derive(Copy, Clone, RustcEncodable, RustcDecodable)] diff --git a/src/librustc_mir/borrow_check/mod.rs b/src/librustc_mir/borrow_check/mod.rs index de27aec2b29..05b396681ac 100644 --- a/src/librustc_mir/borrow_check/mod.rs +++ b/src/librustc_mir/borrow_check/mod.rs @@ -13,7 +13,7 @@ use rustc::mir::{ ClearCrossCrate, Local, Location, Body, Mutability, Operand, Place, PlaceBase, PlaceRef, Static, StaticKind }; -use rustc::mir::{Field, Projection, ProjectionElem, Rvalue, Statement, StatementKind}; +use rustc::mir::{Field, Projection, ProjectionElem, Promoted, Rvalue, Statement, StatementKind}; use rustc::mir::{Terminator, TerminatorKind}; use rustc::ty::query::Providers; use rustc::ty::{self, TyCtxt}; @@ -22,6 +22,7 @@ use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, Level}; use rustc_data_structures::bit_set::BitSet; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::graph::dominators::Dominators; +use rustc_data_structures::indexed_vec::IndexVec; use smallvec::SmallVec; use std::collections::BTreeMap; @@ -86,12 +87,13 @@ pub fn provide(providers: &mut Providers<'_>) { } fn mir_borrowck(tcx: TyCtxt<'_>, def_id: DefId) -> BorrowCheckResult<'_> { - let input_body = tcx.mir_validated(def_id); + let (input_body, promoted) = tcx.mir_validated(def_id); debug!("run query mir_borrowck: {}", tcx.def_path_str(def_id)); let opt_closure_req = tcx.infer_ctxt().enter(|infcx| { let input_body: &Body<'_> = &input_body.borrow(); - do_mir_borrowck(&infcx, input_body, def_id) + let promoted: &IndexVec<_, _> = &promoted.borrow(); + do_mir_borrowck(&infcx, input_body, promoted, def_id) }); debug!("mir_borrowck done"); @@ -101,6 +103,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def_id: DefId) -> BorrowCheckResult<'_> { fn do_mir_borrowck<'a, 'tcx>( infcx: &InferCtxt<'a, 'tcx>, input_body: &Body<'tcx>, + input_promoted: &IndexVec>, def_id: DefId, ) -> BorrowCheckResult<'tcx> { debug!("do_mir_borrowck(def_id = {:?})", def_id); @@ -147,7 +150,8 @@ fn do_mir_borrowck<'a, 'tcx>( // be modified (in place) to contain non-lexical lifetimes. It // will have a lifetime tied to the inference context. let mut body: Body<'tcx> = input_body.clone(); - let free_regions = nll::replace_regions_in_mir(infcx, def_id, param_env, &mut body); + let mut promoted: IndexVec> = input_promoted.clone(); + let free_regions = nll::replace_regions_in_mir(infcx, def_id, param_env, &mut body, &mut promoted); let body = &body; // no further changes let location_table = &LocationTable::new(body); @@ -184,6 +188,7 @@ fn do_mir_borrowck<'a, 'tcx>( def_id, free_regions, body, + &promoted, &upvars, location_table, param_env, diff --git a/src/librustc_mir/borrow_check/nll/mod.rs b/src/librustc_mir/borrow_check/nll/mod.rs index d65cdde303c..11ec154e5b5 100644 --- a/src/librustc_mir/borrow_check/nll/mod.rs +++ b/src/librustc_mir/borrow_check/nll/mod.rs @@ -11,8 +11,9 @@ use crate::transform::MirSource; use crate::borrow_check::Upvar; use rustc::hir::def_id::DefId; use rustc::infer::InferCtxt; -use rustc::mir::{ClosureOutlivesSubject, ClosureRegionRequirements, Local, Body}; +use rustc::mir::{ClosureOutlivesSubject, ClosureRegionRequirements, Local, Body, Promoted}; use rustc::ty::{self, RegionKind, RegionVid}; +use rustc_data_structures::indexed_vec::IndexVec; use rustc_errors::Diagnostic; use std::fmt::Debug; use std::env; @@ -52,6 +53,7 @@ pub(in crate::borrow_check) fn replace_regions_in_mir<'cx, 'tcx>( def_id: DefId, param_env: ty::ParamEnv<'tcx>, body: &mut Body<'tcx>, + promoted: &mut IndexVec>, ) -> UniversalRegions<'tcx> { debug!("replace_regions_in_mir(def_id={:?})", def_id); @@ -59,7 +61,7 @@ pub(in crate::borrow_check) fn replace_regions_in_mir<'cx, 'tcx>( let universal_regions = UniversalRegions::new(infcx, def_id, param_env); // Replace all remaining regions with fresh inference variables. - renumber::renumber_mir(infcx, body); + renumber::renumber_mir(infcx, body, promoted); let source = MirSource::item(def_id); mir_util::dump_mir(infcx.tcx, None, "renumber", &0, source, body, |_, _| Ok(())); @@ -75,6 +77,7 @@ pub(in crate::borrow_check) fn compute_regions<'cx, 'tcx>( def_id: DefId, universal_regions: UniversalRegions<'tcx>, body: &Body<'tcx>, + promoted: &IndexVec>, upvars: &[Upvar], location_table: &LocationTable, param_env: ty::ParamEnv<'tcx>, @@ -105,6 +108,7 @@ pub(in crate::borrow_check) fn compute_regions<'cx, 'tcx>( infcx, param_env, body, + promoted, def_id, &universal_regions, location_table, diff --git a/src/librustc_mir/borrow_check/nll/renumber.rs b/src/librustc_mir/borrow_check/nll/renumber.rs index c1d1185cf17..4e3ffb7af16 100644 --- a/src/librustc_mir/borrow_check/nll/renumber.rs +++ b/src/librustc_mir/borrow_check/nll/renumber.rs @@ -1,16 +1,18 @@ use rustc::ty::subst::SubstsRef; use rustc::ty::{self, ClosureSubsts, GeneratorSubsts, Ty, TypeFoldable}; -use rustc::mir::{Location, Body}; +use rustc::mir::{Location, Body, Promoted}; use rustc::mir::visit::{MutVisitor, TyContext}; use rustc::infer::{InferCtxt, NLLRegionVariableOrigin}; +use rustc_data_structures::indexed_vec::IndexVec; /// Replaces all free regions appearing in the MIR with fresh /// inference variables, returning the number of variables created. -pub fn renumber_mir<'tcx>(infcx: &InferCtxt<'_, 'tcx>, body: &mut Body<'tcx>) { +pub fn renumber_mir<'tcx>(infcx: &InferCtxt<'_, 'tcx>, body: &mut Body<'tcx>, promoted: &mut IndexVec>) { debug!("renumber_mir()"); debug!("renumber_mir: body.arg_count={:?}", body.arg_count); let mut visitor = NLLVisitor { infcx }; + visitor.visit_promoted(promoted); visitor.visit_body(body); } @@ -41,17 +43,16 @@ impl<'a, 'tcx> NLLVisitor<'a, 'tcx> { { renumber_regions(self.infcx, value) } + + fn visit_promoted(&mut self, promoted: &mut IndexVec>) { + debug!("visiting promoted mir"); + for body in promoted.iter_mut() { + self.visit_body(body); + } + } } impl<'a, 'tcx> MutVisitor<'tcx> for NLLVisitor<'a, 'tcx> { - fn visit_body(&mut self, body: &mut Body<'tcx>) { - for promoted in body.promoted.iter_mut() { - self.visit_body(promoted); - } - - self.super_body(body); - } - fn visit_ty(&mut self, ty: &mut Ty<'tcx>, ty_context: TyContext) { debug!("visit_ty(ty={:?}, ty_context={:?})", ty, ty_context); diff --git a/src/librustc_mir/borrow_check/nll/type_check/mod.rs b/src/librustc_mir/borrow_check/nll/type_check/mod.rs index 9ff0c6ca6a5..4b4477756ba 100644 --- a/src/librustc_mir/borrow_check/nll/type_check/mod.rs +++ b/src/librustc_mir/borrow_check/nll/type_check/mod.rs @@ -112,6 +112,7 @@ pub(crate) fn type_check<'tcx>( infcx: &InferCtxt<'_, 'tcx>, param_env: ty::ParamEnv<'tcx>, body: &Body<'tcx>, + promoted: &IndexVec>, mir_def_id: DefId, universal_regions: &Rc>, location_table: &LocationTable, @@ -157,6 +158,7 @@ pub(crate) fn type_check<'tcx>( mir_def_id, param_env, body, + promoted, ®ion_bound_pairs, implicit_region_bound, &mut borrowck_context, @@ -180,6 +182,7 @@ fn type_check_internal<'a, 'tcx, R>( mir_def_id: DefId, param_env: ty::ParamEnv<'tcx>, body: &'a Body<'tcx>, + promoted: &'a IndexVec>, region_bound_pairs: &'a RegionBoundPairs<'tcx>, implicit_region_bound: ty::Region<'tcx>, borrowck_context: &'a mut BorrowCheckContext<'a, 'tcx>, @@ -197,7 +200,7 @@ fn type_check_internal<'a, 'tcx, R>( universal_region_relations, ); let errors_reported = { - let mut verifier = TypeVerifier::new(&mut checker, body); + let mut verifier = TypeVerifier::new(&mut checker, body, promoted); verifier.visit_body(body); verifier.errors_reported }; @@ -254,6 +257,7 @@ enum FieldAccessError { struct TypeVerifier<'a, 'b, 'tcx> { cx: &'a mut TypeChecker<'b, 'tcx>, body: &'b Body<'tcx>, + promoted: &'b IndexVec>, last_span: Span, mir_def_id: DefId, errors_reported: bool, @@ -380,9 +384,10 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> { } impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { - fn new(cx: &'a mut TypeChecker<'b, 'tcx>, body: &'b Body<'tcx>) -> Self { + fn new(cx: &'a mut TypeChecker<'b, 'tcx>, body: &'b Body<'tcx>, promoted: &'b IndexVec>) -> Self { TypeVerifier { body, + promoted, mir_def_id: cx.mir_def_id, cx, last_span: body.span, @@ -442,7 +447,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { match kind { StaticKind::Promoted(promoted) => { if !self.errors_reported { - let promoted_body = &self.body.promoted[*promoted]; + let promoted_body = &self.promoted[*promoted]; self.sanitize_promoted(promoted_body, location); let promoted_ty = promoted_body.return_ty(); diff --git a/src/librustc_mir/build/mod.rs b/src/librustc_mir/build/mod.rs index 4e970aee42c..3e3558fc600 100644 --- a/src/librustc_mir/build/mod.rs +++ b/src/librustc_mir/build/mod.rs @@ -763,7 +763,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.cfg.basic_blocks, self.source_scopes, ClearCrossCrate::Set(self.source_scope_local_data), - IndexVec::new(), yield_ty, self.local_decls, self.canonical_user_type_annotations, diff --git a/src/librustc_mir/const_eval.rs b/src/librustc_mir/const_eval.rs index 76ee76a7456..4ab5c9cc1c4 100644 --- a/src/librustc_mir/const_eval.rs +++ b/src/librustc_mir/const_eval.rs @@ -667,7 +667,7 @@ pub fn const_eval_raw_provider<'tcx>( let res = ecx.load_mir(cid.instance.def); res.map(|body| { if let Some(index) = cid.promoted { - &body.promoted[index] + &tcx.promoted_mir(def_id)[index] } else { body } diff --git a/src/librustc_mir/monomorphize/collector.rs b/src/librustc_mir/monomorphize/collector.rs index 12d763bb791..9d80163f30f 100644 --- a/src/librustc_mir/monomorphize/collector.rs +++ b/src/librustc_mir/monomorphize/collector.rs @@ -185,7 +185,7 @@ use rustc::ty::{self, TypeFoldable, Ty, TyCtxt, GenericParamDefKind, Instance}; use rustc::ty::print::obsolete::DefPathBasedNames; use rustc::ty::adjustment::{CustomCoerceUnsized, PointerCast}; use rustc::session::config::EntryFnType; -use rustc::mir::{self, Location, PlaceBase, Promoted, Static, StaticKind}; +use rustc::mir::{self, Location, PlaceBase, Static, StaticKind}; use rustc::mir::visit::Visitor as MirVisitor; use rustc::mir::mono::{MonoItem, InstantiationMode}; use rustc::mir::interpret::{Scalar, GlobalId, GlobalAlloc, ErrorHandled}; @@ -1222,6 +1222,7 @@ fn collect_neighbours<'tcx>( instance: Instance<'tcx>, output: &mut Vec>, ) { + debug!("collect_neighbours: {:?}", instance.def_id()); let body = tcx.instance_mir(instance.def); MirNeighborCollector { @@ -1230,20 +1231,22 @@ fn collect_neighbours<'tcx>( output, param_substs: instance.substs, }.visit_body(&body); - let param_env = ty::ParamEnv::reveal_all(); - for i in 0..body.promoted.len() { - use rustc_data_structures::indexed_vec::Idx; - let i = Promoted::new(i); - let cid = GlobalId { - instance, - promoted: Some(i), - }; - match tcx.const_eval(param_env.and(cid)) { - Ok(val) => collect_const(tcx, val, instance.substs, output), - Err(ErrorHandled::Reported) => {}, - Err(ErrorHandled::TooGeneric) => span_bug!( - body.promoted[i].span, "collection encountered polymorphic constant", - ), + + if let ty::InstanceDef::Item(def_id) = instance.def { + let param_env = ty::ParamEnv::reveal_all(); + let promoted = tcx.promoted_mir(def_id); + for (promoted, promoted_body) in promoted.iter_enumerated() { + let cid = GlobalId { + instance, + promoted: Some(promoted), + }; + match tcx.const_eval(param_env.and(cid)) { + Ok(val) => collect_const(tcx, val, instance.substs, output), + Err(ErrorHandled::Reported) => {}, + Err(ErrorHandled::TooGeneric) => span_bug!( + promoted_body.span, "collection encountered polymorphic constant", + ), + } } } } diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs index 3e02f6c3725..9d31015f845 100644 --- a/src/librustc_mir/shim.rs +++ b/src/librustc_mir/shim.rs @@ -201,7 +201,6 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option>) SourceScopeData { span: span, parent_scope: None }, 1 ), ClearCrossCrate::Clear, - IndexVec::new(), None, local_decls_for_sig(&sig, span), IndexVec::new(), @@ -369,7 +368,6 @@ impl CloneShimBuilder<'tcx> { SourceScopeData { span: self.span, parent_scope: None }, 1 ), ClearCrossCrate::Clear, - IndexVec::new(), None, self.local_decls, IndexVec::new(), @@ -813,7 +811,6 @@ fn build_call_shim<'tcx>( SourceScopeData { span: span, parent_scope: None }, 1 ), ClearCrossCrate::Clear, - IndexVec::new(), None, local_decls, IndexVec::new(), @@ -900,7 +897,6 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> { SourceScopeData { span: span, parent_scope: None }, 1 ), ClearCrossCrate::Clear, - IndexVec::new(), None, local_decls, IndexVec::new(), diff --git a/src/librustc_mir/transform/const_prop.rs b/src/librustc_mir/transform/const_prop.rs index ac442a496e5..790595b4fee 100644 --- a/src/librustc_mir/transform/const_prop.rs +++ b/src/librustc_mir/transform/const_prop.rs @@ -8,7 +8,7 @@ use rustc::mir::{ AggregateKind, Constant, Location, Place, PlaceBase, Body, Operand, Rvalue, Local, NullOp, UnOp, StatementKind, Statement, LocalKind, Static, StaticKind, TerminatorKind, Terminator, ClearCrossCrate, SourceInfo, BinOp, ProjectionElem, - SourceScope, SourceScopeLocalData, LocalDecl, Promoted, + SourceScope, SourceScopeLocalData, LocalDecl, }; use rustc::mir::visit::{ Visitor, PlaceContext, MutatingUseContext, MutVisitor, NonMutatingUseContext, @@ -64,17 +64,12 @@ impl<'tcx> MirPass<'tcx> for ConstProp { &mut body.source_scope_local_data, ClearCrossCrate::Clear ); - let promoted = std::mem::replace( - &mut body.promoted, - IndexVec::new() - ); let dummy_body = &Body::new( body.basic_blocks().clone(), Default::default(), ClearCrossCrate::Clear, - Default::default(), None, body.local_decls.clone(), Default::default(), @@ -92,22 +87,17 @@ impl<'tcx> MirPass<'tcx> for ConstProp { body, dummy_body, source_scope_local_data, - promoted, tcx, source ); optimization_finder.visit_body(body); // put back the data we stole from `mir` - let (source_scope_local_data, promoted) = optimization_finder.release_stolen_data(); + let source_scope_local_data = optimization_finder.release_stolen_data(); std::mem::replace( &mut body.source_scope_local_data, source_scope_local_data ); - std::mem::replace( - &mut body.promoted, - promoted - ); trace!("ConstProp done for {:?}", source.def_id()); } @@ -124,7 +114,6 @@ struct ConstPropagator<'mir, 'tcx> { param_env: ParamEnv<'tcx>, source_scope_local_data: ClearCrossCrate>, local_decls: IndexVec>, - promoted: IndexVec>, } impl<'mir, 'tcx> LayoutOf for ConstPropagator<'mir, 'tcx> { @@ -155,7 +144,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { body: &Body<'tcx>, dummy_body: &'mir Body<'tcx>, source_scope_local_data: ClearCrossCrate>, - promoted: IndexVec>, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, ) -> ConstPropagator<'mir, 'tcx> { @@ -184,17 +172,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { source_scope_local_data, //FIXME(wesleywiser) we can't steal this because `Visitor::super_visit_body()` needs it local_decls: body.local_decls.clone(), - promoted, } } - fn release_stolen_data( - self, - ) -> ( - ClearCrossCrate>, - IndexVec>, - ) { - (self.source_scope_local_data, self.promoted) + fn release_stolen_data(self) -> ClearCrossCrate> { + self.source_scope_local_data } fn get_const(&self, local: Local) -> Option> { @@ -318,7 +300,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // cannot use `const_eval` here, because that would require having the MIR // for the current function available, but we're producing said MIR right now let res = self.use_ecx(source_info, |this| { - let body = &this.promoted[*promoted]; + let body = &this.tcx.promoted_mir(this.source.def_id())[*promoted]; eval_promoted(this.tcx, cid, body, this.param_env) })?; trace!("evaluated promoted {:?} to {:?}", promoted, res); diff --git a/src/librustc_mir/transform/inline.rs b/src/librustc_mir/transform/inline.rs index 57aac2b0eac..19a8769ce16 100644 --- a/src/librustc_mir/transform/inline.rs +++ b/src/librustc_mir/transform/inline.rs @@ -394,7 +394,7 @@ impl Inliner<'tcx> { let mut local_map = IndexVec::with_capacity(callee_body.local_decls.len()); let mut scope_map = IndexVec::with_capacity(callee_body.source_scopes.len()); - let mut promoted_map = IndexVec::with_capacity(callee_body.promoted.len()); + let promoted_map = IndexVec::with_capacity(self.tcx.promoted_mir(callsite.callee).len()); for mut scope in callee_body.source_scopes.iter().cloned() { if scope.parent_scope.is_none() { @@ -420,9 +420,10 @@ impl Inliner<'tcx> { local_map.push(idx); } - promoted_map.extend( - callee_body.promoted.iter().cloned().map(|p| caller_body.promoted.push(p)) - ); + //TODO fixme + //promoted_map.extend( + // self.tcx.promoted_mir(callsite.callee).iter().cloned().map(|p| caller_body.promoted.push(p)) + //); // If the call is something like `a[*i] = f(i)`, where // `i : &mut usize`, then just duplicating the `a[*i]` diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs index 255635b9333..a78e78331ee 100644 --- a/src/librustc_mir/transform/mod.rs +++ b/src/librustc_mir/transform/mod.rs @@ -184,13 +184,6 @@ pub fn run_passes( }; run_passes(body, None); - - for (index, promoted_body) in body.promoted.iter_enumerated_mut() { - run_passes(promoted_body, Some(index)); - - //Let's make sure we don't miss any nested instances - assert!(promoted_body.promoted.is_empty()) - } } fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal> { @@ -207,7 +200,7 @@ fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal> { tcx.alloc_steal_mir(body) } -fn mir_validated(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx Steal> { +fn mir_validated(tcx: TyCtxt<'tcx>, def_id: DefId) -> (&'tcx Steal>, &'tcx Steal>>) { let hir_id = tcx.hir().as_local_hir_id(def_id).unwrap(); if let hir::BodyOwnerKind::Const = tcx.hir().body_owner_kind(hir_id) { // Ensure that we compute the `mir_const_qualif` for constants at @@ -216,12 +209,14 @@ fn mir_validated(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx Steal> { } let mut body = tcx.mir_const(def_id).steal(); + let qualify_and_promote_pass = qualify_consts::QualifyAndPromoteConstants::default(); run_passes(tcx, &mut body, InstanceDef::Item(def_id), MirPhase::Validated, &[ // What we need to run borrowck etc. - &qualify_consts::QualifyAndPromoteConstants, + &qualify_and_promote_pass, &simplify::SimplifyCfg::new("qualify-consts"), ]); - tcx.alloc_steal_mir(body) + let promoted = qualify_and_promote_pass.promoted.into_inner(); + (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted.unwrap_or_else(|| IndexVec::new()))) } fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { @@ -241,7 +236,8 @@ fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { tcx.ensure().borrowck(def_id); } - let mut body = tcx.mir_validated(def_id).steal(); + let (body, _) = tcx.mir_validated(def_id); + let mut body = body.steal(); run_passes(tcx, &mut body, InstanceDef::Item(def_id), MirPhase::Optimized, &[ // Remove all things only needed by analysis &no_landing_pads::NoLandingPads, @@ -297,6 +293,66 @@ fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { } fn promoted_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx IndexVec> { - let body = tcx.optimized_mir(def_id); - &body.promoted + if tcx.is_constructor(def_id) { + return tcx.intern_promoted(IndexVec::new()); + } + + tcx.ensure().mir_borrowck(def_id); + let (_, promoted) = tcx.mir_validated(def_id); + let mut promoted = promoted.steal(); + + for mut body in promoted.iter_mut() { + run_passes(tcx, &mut body, InstanceDef::Item(def_id), MirPhase::Optimized, &[ + // Remove all things only needed by analysis + &no_landing_pads::NoLandingPads, + &simplify_branches::SimplifyBranches::new("initial"), + &remove_noop_landing_pads::RemoveNoopLandingPads, + &cleanup_post_borrowck::CleanupNonCodegenStatements, + + &simplify::SimplifyCfg::new("early-opt"), + + // These next passes must be executed together + &add_call_guards::CriticalCallEdges, + &elaborate_drops::ElaborateDrops, + &no_landing_pads::NoLandingPads, + // AddMovesForPackedDrops needs to run after drop + // elaboration. + &add_moves_for_packed_drops::AddMovesForPackedDrops, + // AddRetag needs to run after ElaborateDrops, and it needs + // an AllCallEdges pass right before it. Otherwise it should + // run fairly late, but before optimizations begin. + &add_call_guards::AllCallEdges, + &add_retag::AddRetag, + + &simplify::SimplifyCfg::new("elaborate-drops"), + + // No lifetime analysis based on borrowing can be done from here on out. + + // From here on out, regions are gone. + &erase_regions::EraseRegions, + + // Optimizations begin. + &uniform_array_move_out::RestoreSubsliceArrayMoveOut, + &inline::Inline, + + // Lowering generator control-flow and variables + // has to happen before we do anything else to them. + &generator::StateTransform, + + &instcombine::InstCombine, + &const_prop::ConstProp, + &simplify_branches::SimplifyBranches::new("after-const-prop"), + &deaggregator::Deaggregator, + ©_prop::CopyPropagation, + &simplify_branches::SimplifyBranches::new("after-copy-prop"), + &remove_noop_landing_pads::RemoveNoopLandingPads, + &simplify::SimplifyCfg::new("final"), + &simplify::SimplifyLocals, + + &add_call_guards::CriticalCallEdges, + &dump_mir::Marker("PreCodegen"), + ]); + } + + tcx.intern_promoted(promoted) } diff --git a/src/librustc_mir/transform/promote_consts.rs b/src/librustc_mir/transform/promote_consts.rs index 3090b63a7e9..7015e2c087f 100644 --- a/src/librustc_mir/transform/promote_consts.rs +++ b/src/librustc_mir/transform/promote_consts.rs @@ -293,10 +293,10 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { new_temp } - fn promote_candidate(mut self, candidate: Candidate) { + fn promote_candidate(mut self, candidate: Candidate, next_promoted_id: usize) -> Option> { let mut operand = { let promoted = &mut self.promoted; - let promoted_id = Promoted::new(self.source.promoted.len()); + let promoted_id = Promoted::new(next_promoted_id); let mut promoted_place = |ty, span| { promoted.span = span; promoted.local_decls[RETURN_PLACE] = LocalDecl::new_return_place(ty, span); @@ -353,7 +353,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { // a function requiring a constant argument and as that constant value // providing a value whose computation contains another call to a function // requiring a constant argument. - TerminatorKind::Goto { .. } => return, + TerminatorKind::Goto { .. } => return None, _ => bug!() } } @@ -368,7 +368,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { let span = self.promoted.span; self.assign(RETURN_PLACE, Rvalue::Use(operand), span); - self.source.promoted.push(self.promoted); + Some(self.promoted) } } @@ -389,10 +389,12 @@ pub fn promote_candidates<'tcx>( tcx: TyCtxt<'tcx>, mut temps: IndexVec, candidates: Vec, -) { +) -> IndexVec> { // Visit candidates in reverse, in case they're nested. debug!("promote_candidates({:?})", candidates); + let mut promotions = IndexVec::new(); + for candidate in candidates.into_iter().rev() { match candidate { Candidate::Repeat(Location { block, statement_index }) | @@ -426,7 +428,6 @@ pub fn promote_candidates<'tcx>( // memory usage? body.source_scopes.clone(), body.source_scope_local_data.clone(), - IndexVec::new(), None, initial_locals, IndexVec::new(), @@ -440,7 +441,10 @@ pub fn promote_candidates<'tcx>( temps: &mut temps, keep_original: false }; - promoter.promote_candidate(candidate); + + if let Some(promoted) = promoter.promote_candidate(candidate, promotions.len()) { + promotions.push(promoted); + } } // Eliminate assignments to, and drops of promoted temps. @@ -474,4 +478,6 @@ pub fn promote_candidates<'tcx>( _ => {} } } + + promotions } diff --git a/src/librustc_mir/transform/qualify_consts.rs b/src/librustc_mir/transform/qualify_consts.rs index 1fe45a2c424..dd4db479cc0 100644 --- a/src/librustc_mir/transform/qualify_consts.rs +++ b/src/librustc_mir/transform/qualify_consts.rs @@ -25,6 +25,7 @@ use syntax::feature_gate::{emit_feature_err, GateIssue}; use syntax::symbol::sym; use syntax_pos::{Span, DUMMY_SP}; +use std::cell::Cell; use std::fmt; use std::ops::{Deref, Index, IndexMut}; use std::usize; @@ -1570,9 +1571,19 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def_id: DefId) -> (u8, &BitSet) { Checker::new(tcx, def_id, body, Mode::Const).check_const() } -pub struct QualifyAndPromoteConstants; +pub struct QualifyAndPromoteConstants<'tcx> { + pub promoted: Cell>>>, +} -impl<'tcx> MirPass<'tcx> for QualifyAndPromoteConstants { +impl<'tcx> Default for QualifyAndPromoteConstants<'tcx> { + fn default() -> Self { + QualifyAndPromoteConstants { + promoted: Cell::new(None), + } + } +} + +impl<'tcx> MirPass<'tcx> for QualifyAndPromoteConstants<'tcx> { fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut Body<'tcx>) { // There's not really any point in promoting errorful MIR. if body.return_ty().references_error() { @@ -1649,7 +1660,9 @@ impl<'tcx> MirPass<'tcx> for QualifyAndPromoteConstants { }; // Do the actual promotion, now that we know what's viable. - promote_consts::promote_candidates(body, tcx, temps, candidates); + self.promoted.set( + Some(promote_consts::promote_candidates(body, tcx, temps, candidates)) + ); } else { if !body.control_flow_destroyed.is_empty() { let mut locals = body.vars_iter();