From 1dd9c3e52a78399ca071728c76fafcd9524f793e Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Mon, 1 May 2017 16:03:05 -0400 Subject: [PATCH] support inlining by asking for optimizer mir for callees I tested this with it enabled 100% of the time, and we were able to run mir-opt tests successfully. --- src/librustc_driver/driver.rs | 2 +- src/librustc_mir/queries.rs | 5 + src/librustc_mir/transform/inline.rs | 205 +++++++++------------------ 3 files changed, 70 insertions(+), 142 deletions(-) diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index a08fd1f9bf6..f80d6473dcc 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -928,7 +928,7 @@ pub fn phase_3_run_analysis_passes<'tcx, F, R>(sess: &'tcx Session, passes.push_pass(MIR_OPTIMIZED, mir::transform::simplify::SimplifyCfg::new("elaborate-drops")); // No lifetime analysis based on borrowing can be done from here on out. - // TODO passes.push_pass(MIR_OPTIMIZED, mir::transform::inline::Inline); + passes.push_pass(MIR_OPTIMIZED, mir::transform::inline::Inline); passes.push_pass(MIR_OPTIMIZED, mir::transform::instcombine::InstCombine); passes.push_pass(MIR_OPTIMIZED, mir::transform::deaggregator::Deaggregator); passes.push_pass(MIR_OPTIMIZED, mir::transform::copy_prop::CopyPropagation); diff --git a/src/librustc_mir/queries.rs b/src/librustc_mir/queries.rs index 980663ec675..b9d6e527e55 100644 --- a/src/librustc_mir/queries.rs +++ b/src/librustc_mir/queries.rs @@ -38,10 +38,15 @@ pub fn provide(providers: &mut Providers) { mir_const, mir_validated, optimized_mir, + is_item_mir_available, ..*providers }; } +fn is_item_mir_available<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> bool { + tcx.mir_keys(def_id.krate).contains(&def_id) +} + /// Finds the full set of def-ids within the current crate that have /// MIR associated with them. fn mir_keys<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, krate: CrateNum) diff --git a/src/librustc_mir/transform/inline.rs b/src/librustc_mir/transform/inline.rs index caa2aa74ede..614b5e2cc3f 100644 --- a/src/librustc_mir/transform/inline.rs +++ b/src/librustc_mir/transform/inline.rs @@ -14,27 +14,20 @@ use rustc::hir::def_id::DefId; use rustc_data_structures::bitvec::BitVector; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; -use rustc_data_structures::graph; -use rustc::dep_graph::DepNode; use rustc::mir::*; -use rustc::mir::transform::{MirSource, PassId}; +use rustc::mir::transform::{MirPass, MirSource}; use rustc::mir::visit::*; use rustc::traits; use rustc::ty::{self, Ty, TyCtxt}; -use rustc::ty::maps::Multi; -use rustc::ty::steal::Steal; use rustc::ty::subst::{Subst,Substs}; -use rustc::util::nodemap::{DefIdSet}; +use std::collections::VecDeque; use super::simplify::{remove_dead_blocks, CfgSimplifier}; use syntax::{attr}; use syntax::abi::Abi; -use callgraph; -use transform::interprocedural::InterproceduralCx; - const DEFAULT_THRESHOLD: usize = 50; const HINT_THRESHOLD: usize = 100; @@ -45,101 +38,63 @@ const UNKNOWN_SIZE_COST: usize = 10; pub struct Inline; -pub trait Pass { - fn run_pass<'a, 'tcx: 'a>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) - -> Multi>>; -} - -impl Pass for Inline { - fn run_pass<'a, 'tcx: 'a>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) - -> Multi>> { - let mut cx = InterproceduralCx::new(tcx); - - let callgraph = callgraph::CallGraph::build(&mut cx); - - let mut inliner = Inliner { tcx }; - - for scc in callgraph.scc_iter() { - inliner.inline_scc(&mut cx, &callgraph, &scc); - } - - Multi::from(cx.into_local_mirs()) - } -} - -struct Inliner<'a, 'tcx: 'a> { - tcx: TyCtxt<'a, 'tcx, 'tcx>, -} - #[derive(Copy, Clone)] struct CallSite<'tcx> { - caller: DefId, callee: DefId, substs: &'tcx Substs<'tcx>, bb: BasicBlock, location: SourceInfo, } -impl<'a, 'tcx> Inliner<'a, 'tcx> { - fn inline_scc(&mut self, - cx: &mut InterproceduralCx<'a, 'tcx>, - callgraph: &callgraph::CallGraph, - scc: &[graph::NodeIndex]) -> bool { - let tcx = self.tcx; - let mut callsites = Vec::new(); - let mut in_scc = DefIdSet(); - - let mut inlined_into = DefIdSet(); - - for &node in scc { - let def_id = callgraph.def_id(node); - - // Don't inspect functions from other crates - let id = if let Some(id) = tcx.hir.as_local_node_id(def_id) { - id - } else { - continue; - }; - let src = MirSource::from_node(tcx, id); - if let MirSource::Fn(_) = src { - if let Some(mir) = cx.ensure_mir_and_read(def_id) { - for (bb, bb_data) in mir.basic_blocks().iter_enumerated() { - // Don't inline calls that are in cleanup blocks. - if bb_data.is_cleanup { continue; } - - // Only consider direct calls to functions - let terminator = bb_data.terminator(); - if let TerminatorKind::Call { - func: Operand::Constant(ref f), .. } = terminator.kind { - if let ty::TyFnDef(callee_def_id, substs, _) = f.ty.sty { - callsites.push(CallSite { - caller: def_id, - callee: callee_def_id, - substs: substs, - bb: bb, - location: terminator.source_info - }); - } - } - } - - in_scc.insert(def_id); - } - } +impl MirPass for Inline { + fn run_pass<'a, 'tcx>(&self, + tcx: TyCtxt<'a, 'tcx, 'tcx>, + source: MirSource, + mir: &mut Mir<'tcx>) { + if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 { + Inliner { tcx, source }.run_pass(mir); } + } +} - // Move callsites that are in the the SCC to the end so - // they're inlined after calls to outside the SCC - let mut first_call_in_scc = callsites.len(); +struct Inliner<'a, 'tcx: 'a> { + tcx: TyCtxt<'a, 'tcx, 'tcx>, + source: MirSource, +} - let mut i = 0; - while i < first_call_in_scc { - let f = callsites[i].caller; - if in_scc.contains(&f) { - first_call_in_scc -= 1; - callsites.swap(i, first_call_in_scc); - } else { - i += 1; +impl<'a, 'tcx> Inliner<'a, 'tcx> { + fn run_pass(&self, caller_mir: &mut Mir<'tcx>) { + // Keep a queue of callsites to try inlining on. We take + // advantage of the fact that queries detect cycles here to + // allow us to try and fetch the fully optimized MIR of a + // call; if it succeeds, we can inline it and we know that + // they do not call us. Otherwise, we just don't try to + // inline. + // + // We use a queue so that we inline "broadly" before we inline + // in depth. It is unclear if this is the current heuristic. + + let mut callsites = VecDeque::new(); + + // Only do inlining into fn bodies. + if let MirSource::Fn(_) = self.source { + for (bb, bb_data) in caller_mir.basic_blocks().iter_enumerated() { + // Don't inline calls that are in cleanup blocks. + if bb_data.is_cleanup { continue; } + + // Only consider direct calls to functions + let terminator = bb_data.terminator(); + if let TerminatorKind::Call { + func: Operand::Constant(ref f), .. } = terminator.kind { + if let ty::TyFnDef(callee_def_id, substs, _) = f.ty.sty { + callsites.push_back(CallSite { + callee: callee_def_id, + substs: substs, + bb: bb, + location: terminator.source_info + }); + } + } } } @@ -148,37 +103,27 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { loop { local_change = false; - let mut csi = 0; - while csi < callsites.len() { - let callsite = callsites[csi]; - csi += 1; + while let Some(callsite) = callsites.pop_front() { + if !self.tcx.is_item_mir_available(callsite.callee) { + continue; + } - let _task = tcx.dep_graph.in_task(DepNode::Mir(callsite.caller)); - tcx.dep_graph.write(DepNode::Mir(callsite.caller)); - - let callee_mir = { - if let Some(callee_mir) = cx.ensure_mir_and_read(callsite.callee) { - if !self.should_inline(callsite, &callee_mir) { - continue; - } - - callee_mir.subst(tcx, callsite.substs) - } else { - continue; + let callee_mir = match ty::queries::optimized_mir::try_get(self.tcx, + callsite.location.span, + callsite.callee) { + Ok(ref callee_mir) if self.should_inline(callsite, callee_mir) => { + callee_mir.subst(self.tcx, callsite.substs) } + _ => continue, }; - let caller_mir = cx.mir_mut(callsite.caller); - let start = caller_mir.basic_blocks().len(); if !self.inline_call(callsite, caller_mir, callee_mir) { continue; } - inlined_into.insert(callsite.caller); - // Add callsites from inlined function for (bb, bb_data) in caller_mir.basic_blocks().iter_enumerated().skip(start) { // Only consider direct calls to functions @@ -188,8 +133,7 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { if let ty::TyFnDef(callee_def_id, substs, _) = f.ty.sty { // Don't inline the same function multiple times. if callsite.callee != callee_def_id { - callsites.push(CallSite { - caller: callsite.caller, + callsites.push_back(CallSite { callee: callee_def_id, substs: substs, bb: bb, @@ -200,13 +144,6 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { } } - csi -= 1; - if scc.len() == 1 { - callsites.swap_remove(csi); - } else { - callsites.remove(csi); - } - local_change = true; changed = true; } @@ -216,18 +153,12 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { } } - // Simplify functions we inlined into. - for def_id in inlined_into { - let _task = tcx.dep_graph.in_task(DepNode::Mir(def_id)); - tcx.dep_graph.write(DepNode::Mir(def_id)); - - let caller_mir = cx.mir_mut(def_id); - - debug!("Running simplify cfg on {:?}", def_id); + // Simplify if we inlined anything. + if changed { + debug!("Running simplify cfg on {:?}", self.source); CfgSimplifier::new(caller_mir).simplify(); remove_dead_blocks(caller_mir); } - changed } fn should_inline(&self, @@ -286,8 +217,7 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { // FIXME: Give a bonus to functions with only a single caller - let id = tcx.hir.as_local_node_id(callsite.caller).expect("Caller not local"); - let param_env = ty::ParameterEnvironment::for_item(tcx, id); + let param_env = ty::ParameterEnvironment::for_item(tcx, self.source.item_id()); let mut first_block = true; let mut cost = 0; @@ -390,18 +320,11 @@ impl<'a, 'tcx> Inliner<'a, 'tcx> { callsite: CallSite<'tcx>, caller_mir: &mut Mir<'tcx>, mut callee_mir: Mir<'tcx>) -> bool { - // Don't inline a function into itself - if callsite.caller == callsite.callee { return false; } - - let _task = self.tcx.dep_graph.in_task(DepNode::Mir(callsite.caller)); - - let terminator = caller_mir[callsite.bb].terminator.take().unwrap(); match terminator.kind { // FIXME: Handle inlining of diverging calls TerminatorKind::Call { args, destination: Some(destination), cleanup, .. } => { - - debug!("Inlined {:?} into {:?}", callsite.callee, callsite.caller); + debug!("Inlined {:?} into {:?}", callsite.callee, self.source); let is_box_free = Some(callsite.callee) == self.tcx.lang_items.box_free_fn();