Add a Rayon thread pool

src/librustc/session/mod.rs

@@ -869,10 +869,16 @@ impl Session {
         ret
     }
 
+    /// Returns the number of query threads that should be used for this
+    /// compilation
+    pub fn query_threads_from_opts(opts: &config::Options) -> usize {
+        opts.debugging_opts.query_threads.unwrap_or(1)
+    }
+
     /// Returns the number of query threads that should be used for this
     /// compilation
     pub fn query_threads(&self) -> usize {
-        self.opts.debugging_opts.query_threads.unwrap_or(1)
+        Self::query_threads_from_opts(&self.opts)
     }
 
     /// Returns the number of codegen units that should be used for this

src/librustc/ty/context.rs

@@ -1800,9 +1800,11 @@ pub mod tls {
     /// in librustc otherwise. It is used to when diagnostic messages are
     /// emitted and stores them in the current query, if there is one.
     fn track_diagnostic(diagnostic: &Diagnostic) {
-        with_context(|context| {
+        with_context_opt(|icx| {
-            if let Some(ref query) = context.query {
+            if let Some(icx) = icx {
-                query.diagnostics.lock().push(diagnostic.clone());
+                if let Some(ref query) = icx.query {
+                    query.diagnostics.lock().push(diagnostic.clone());
+                }
             }
         })
     }

src/librustc_driver/Cargo.toml

@@ -13,6 +13,8 @@ arena = { path = "../libarena" }
 graphviz = { path = "../libgraphviz" }
 log = "0.4"
 env_logger = { version = "0.5", default-features = false }
+rustc-rayon = "0.1.0"
+scoped-tls = { version = "0.1.1", features = ["nightly"] }
 rustc = { path = "../librustc" }
 rustc_allocator = { path = "../librustc_allocator" }
 rustc_target = { path = "../librustc_target" }

src/librustc_driver/driver.rs

@@ -49,7 +49,7 @@ use std::fs;
 use std::io::{self, Write};
 use std::iter;
 use std::path::{Path, PathBuf};
-use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::{self, Lrc};
 use std::sync::mpsc;
 use syntax::{self, ast, attr, diagnostics, visit};
 use syntax::ext::base::ExtCtxt;
@@ -64,6 +64,51 @@ use pretty::ReplaceBodyWithLoop;
 
 use profile;
 
+#[cfg(not(parallel_queries))]
+pub fn spawn_thread_pool<F: FnOnce(config::Options) -> R + sync::Send, R: sync::Send>(
+    opts: config::Options,
+    f: F
+) -> R {
+    f(opts)
+}
+
+#[cfg(parallel_queries)]
+pub fn spawn_thread_pool<F: FnOnce(config::Options) -> R + sync::Send, R: sync::Send>(
+    opts: config::Options,
+    f: F
+) -> R {
+    use syntax;
+    use syntax_pos;
+    use rayon::{ThreadPoolBuilder, ThreadPool};
+
+    let config = ThreadPoolBuilder::new().num_threads(Session::query_threads_from_opts(&opts))
+                                         .stack_size(16 * 1024 * 1024);
+
+    let with_pool = move |pool: &ThreadPool| {
+        pool.install(move || f(opts))
+    };
+
+    syntax::GLOBALS.with(|syntax_globals| {
+        syntax_pos::GLOBALS.with(|syntax_pos_globals| {
+            // The main handler run for each Rayon worker thread and sets up
+            // the thread local rustc uses. syntax_globals and syntax_pos_globals are
+            // captured and set on the new threads. ty::tls::with_thread_locals sets up
+            // thread local callbacks from libsyntax
+            let main_handler = move |worker: &mut FnMut()| {
+                syntax::GLOBALS.set(syntax_globals, || {
+                    syntax_pos::GLOBALS.set(syntax_pos_globals, || {
+                        ty::tls::with_thread_locals(|| {
+                            worker()
+                        })
+                    })
+                })
+            };
+
+            ThreadPool::scoped_pool(config, main_handler, with_pool).unwrap()
+        })
+    })
+}
+
 pub fn compile_input(
     trans: Box<TransCrate>,
     sess: &Session,

src/librustc_driver/lib.rs

@@ -35,6 +35,7 @@ extern crate graphviz;
 extern crate env_logger;
 #[cfg(unix)]
 extern crate libc;
+extern crate rustc_rayon as rayon;
 extern crate rustc;
 extern crate rustc_allocator;
 extern crate rustc_target;
@@ -53,6 +54,7 @@ extern crate rustc_save_analysis;
 extern crate rustc_traits;
 extern crate rustc_trans_utils;
 extern crate rustc_typeck;
+extern crate scoped_tls;
 extern crate serialize;
 #[macro_use]
 extern crate log;
@@ -66,7 +68,7 @@ use pretty::{PpMode, UserIdentifiedItem};
 use rustc_resolve as resolve;
 use rustc_save_analysis as save;
 use rustc_save_analysis::DumpHandler;
-use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::{self, Lrc};
 use rustc_data_structures::OnDrop;
 use rustc::session::{self, config, Session, build_session, CompileResult};
 use rustc::session::CompileIncomplete;
@@ -450,22 +452,33 @@ fn get_trans_sysroot(backend_name: &str) -> fn() -> Box<TransCrate> {
 // See comments on CompilerCalls below for details about the callbacks argument.
 // The FileLoader provides a way to load files from sources other than the file system.
 pub fn run_compiler<'a>(args: &[String],
-                        callbacks: &mut CompilerCalls<'a>,
+                        callbacks: &mut (CompilerCalls<'a> + sync::Send),
                         file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
                         emitter_dest: Option<Box<Write + Send>>)
                         -> (CompileResult, Option<Session>)
 {
     syntax::with_globals(|| {
-        run_compiler_impl(args, callbacks, file_loader, emitter_dest)
+        let matches = match handle_options(args) {
+            Some(matches) => matches,
+            None => return (Ok(()), None),
+        };
+
+        let (sopts, cfg) = config::build_session_options_and_crate_config(&matches);
+
+        driver::spawn_thread_pool(sopts, |sopts| {
+            run_compiler_with_pool(matches, sopts, cfg, callbacks, file_loader, emitter_dest)
+        })
     })
 }
 
-fn run_compiler_impl<'a>(args: &[String],
+fn run_compiler_with_pool<'a>(
-                         callbacks: &mut CompilerCalls<'a>,
+    matches: getopts::Matches,
-                         file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
+    sopts: config::Options,
-                         emitter_dest: Option<Box<Write + Send>>)
+    cfg: ast::CrateConfig,
-                         -> (CompileResult, Option<Session>)
+    callbacks: &mut (CompilerCalls<'a> + sync::Send),
-{
+    file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
+    emitter_dest: Option<Box<Write + Send>>
+) -> (CompileResult, Option<Session>) {
     macro_rules! do_or_return {($expr: expr, $sess: expr) => {
         match $expr {
             Compilation::Stop => return (Ok(()), $sess),
@@ -473,13 +486,6 @@ fn run_compiler_impl<'a>(args: &[String],
         }
     }}
 
-    let matches = match handle_options(args) {
-        Some(matches) => matches,
-        None => return (Ok(()), None),
-    };
-
-    let (sopts, cfg) = config::build_session_options_and_crate_config(&matches);
-
     let descriptions = diagnostics_registry();
 
     do_or_return!(callbacks.early_callback(&matches,

src/librustc_driver/test.rs

@@ -99,20 +99,25 @@ fn test_env<F>(source_string: &str,
     where F: FnOnce(Env)
 {
     syntax::with_globals(|| {
-        test_env_impl(source_string, args, body)
+        let mut options = config::basic_options();
+        options.debugging_opts.verbose = true;
+        options.unstable_features = UnstableFeatures::Allow;
+
+        driver::spawn_thread_pool(options, |options| {
+            test_env_with_pool(options, source_string, args, body)
+        })
     });
 }
 
-fn test_env_impl<F>(source_string: &str,
+fn test_env_with_pool<F>(
-                    (emitter, expected_err_count): (Box<Emitter + sync::Send>, usize),
+    options: config::Options,
-                    body: F)
+    source_string: &str,
+    (emitter, expected_err_count): (Box<Emitter + sync::Send>, usize),
+    body: F
+)
     where F: FnOnce(Env)
 {
-    let mut options = config::basic_options();
-    options.debugging_opts.verbose = true;
-    options.unstable_features = UnstableFeatures::Allow;
     let diagnostic_handler = errors::Handler::with_emitter(true, false, emitter);
-
     let sess = session::build_session_(options,
                                        None,
                                        diagnostic_handler,

src/librustdoc/clean/mod.rs

@@ -697,7 +697,7 @@ impl<'a> FromIterator<&'a DocFragment> for String {
 pub struct Attributes {
     pub doc_strings: Vec<DocFragment>,
     pub other_attrs: Vec<ast::Attribute>,
-    pub cfg: Option<Rc<Cfg>>,
+    pub cfg: Option<Arc<Cfg>>,
     pub span: Option<syntax_pos::Span>,
     /// map from Rust paths to resolved defs and potential URL fragments
     pub links: Vec<(String, Option<DefId>, Option<String>)>,
@@ -848,7 +848,7 @@ impl Attributes {
         Attributes {
             doc_strings,
             other_attrs,
-            cfg: if cfg == Cfg::True { None } else { Some(Rc::new(cfg)) },
+            cfg: if cfg == Cfg::True { None } else { Some(Arc::new(cfg)) },
             span: sp,
             links: vec![],
         }

src/librustdoc/core.rs

@@ -161,161 +161,162 @@ pub fn run_core(search_paths: SearchPaths,
         edition,
         ..config::basic_options().clone()
     };
-
+    driver::spawn_thread_pool(sessopts, move |sessopts| {
-    let codemap = Lrc::new(codemap::CodeMap::new(sessopts.file_path_mapping()));
+        let codemap = Lrc::new(codemap::CodeMap::new(sessopts.file_path_mapping()));
-    let emitter: Box<dyn Emitter + sync::Send> = match error_format {
+        let emitter: Box<dyn Emitter + sync::Send> = match error_format {
-        ErrorOutputType::HumanReadable(color_config) => Box::new(
+            ErrorOutputType::HumanReadable(color_config) => Box::new(
-            EmitterWriter::stderr(
+                EmitterWriter::stderr(
-                color_config,
+                    color_config,
-                Some(codemap.clone()),
+                    Some(codemap.clone()),
-                false,
+                    false,
-                sessopts.debugging_opts.teach,
+                    sessopts.debugging_opts.teach,
-            ).ui_testing(sessopts.debugging_opts.ui_testing)
+                ).ui_testing(sessopts.debugging_opts.ui_testing)
-        ),
+            ),
-        ErrorOutputType::Json(pretty) => Box::new(
+            ErrorOutputType::Json(pretty) => Box::new(
-            JsonEmitter::stderr(
+                JsonEmitter::stderr(
-                None,
+                    None,
-                codemap.clone(),
+                    codemap.clone(),
-                pretty,
+                    pretty,
-                sessopts.debugging_opts.suggestion_applicability,
+                    sessopts.debugging_opts.suggestion_applicability,
-            ).ui_testing(sessopts.debugging_opts.ui_testing)
+                ).ui_testing(sessopts.debugging_opts.ui_testing)
-        ),
+            ),
-        ErrorOutputType::Short(color_config) => Box::new(
+            ErrorOutputType::Short(color_config) => Box::new(
-            EmitterWriter::stderr(color_config, Some(codemap.clone()), true, false)
+                EmitterWriter::stderr(color_config, Some(codemap.clone()), true, false)
-        ),
+            ),
-    };
-
-    let diagnostic_handler = errors::Handler::with_emitter_and_flags(
-        emitter,
-        errors::HandlerFlags {
-            can_emit_warnings: true,
-            treat_err_as_bug: false,
-            external_macro_backtrace: false,
-            ..Default::default()
-        },
-    );
-
-    let mut sess = session::build_session_(
-        sessopts, cpath, diagnostic_handler, codemap,
-    );
-    let trans = rustc_driver::get_trans(&sess);
-    let cstore = Rc::new(CStore::new(trans.metadata_loader()));
-    rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
-
-    let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs));
-    target_features::add_configuration(&mut cfg, &sess, &*trans);
-    sess.parse_sess.config = cfg;
-
-    let control = &driver::CompileController::basic();
-
-    let krate = panictry!(driver::phase_1_parse_input(control, &sess, &input));
-
-    let name = ::rustc_trans_utils::link::find_crate_name(Some(&sess), &krate.attrs, &input);
-
-    let mut crate_loader = CrateLoader::new(&sess, &cstore, &name);
-
-    let resolver_arenas = resolve::Resolver::arenas();
-    let result = driver::phase_2_configure_and_expand_inner(&sess,
-                                                      &cstore,
-                                                      krate,
-                                                      None,
-                                                      &name,
-                                                      None,
-                                                      resolve::MakeGlobMap::No,
-                                                      &resolver_arenas,
-                                                      &mut crate_loader,
-                                                      |_| Ok(()));
-    let driver::InnerExpansionResult {
-        mut hir_forest,
-        resolver,
-        ..
-    } = abort_on_err(result, &sess);
-
-    // We need to hold on to the complete resolver, so we clone everything
-    // for the analysis passes to use. Suboptimal, but necessary in the
-    // current architecture.
-    let defs = resolver.definitions.clone();
-    let resolutions = ty::Resolutions {
-        freevars: resolver.freevars.clone(),
-        export_map: resolver.export_map.clone(),
-        trait_map: resolver.trait_map.clone(),
-        maybe_unused_trait_imports: resolver.maybe_unused_trait_imports.clone(),
-        maybe_unused_extern_crates: resolver.maybe_unused_extern_crates.clone(),
-    };
-    let analysis = ty::CrateAnalysis {
-        access_levels: Lrc::new(AccessLevels::default()),
-        name: name.to_string(),
-        glob_map: if resolver.make_glob_map { Some(resolver.glob_map.clone()) } else { None },
-    };
-
-    let arenas = AllArenas::new();
-    let hir_map = hir_map::map_crate(&sess, &*cstore, &mut hir_forest, &defs);
-    let output_filenames = driver::build_output_filenames(&input,
-                                                          &None,
-                                                          &None,
-                                                          &[],
-                                                          &sess);
-
-    let resolver = RefCell::new(resolver);
-
-    abort_on_err(driver::phase_3_run_analysis_passes(&*trans,
-                                                     control,
-                                                     &sess,
-                                                     &*cstore,
-                                                     hir_map,
-                                                     analysis,
-                                                     resolutions,
-                                                     &arenas,
-                                                     &name,
-                                                     &output_filenames,
-                                                     |tcx, analysis, _, result| {
-        if let Err(_) = result {
-            sess.fatal("Compilation failed, aborting rustdoc");
-        }
-
-        let ty::CrateAnalysis { access_levels, .. } = analysis;
-
-        // Convert from a NodeId set to a DefId set since we don't always have easy access
-        // to the map from defid -> nodeid
-        let access_levels = AccessLevels {
-            map: access_levels.map.iter()
-                                  .map(|(&k, &v)| (tcx.hir.local_def_id(k), v))
-                                  .collect()
         };
 
-        let send_trait = if crate_name == Some("core".to_string()) {
+        let diagnostic_handler = errors::Handler::with_emitter_and_flags(
-            clean::get_trait_def_id(&tcx, &["marker", "Send"], true)
+            emitter,
-        } else {
+            errors::HandlerFlags {
-            clean::get_trait_def_id(&tcx, &["core", "marker", "Send"], false)
+                can_emit_warnings: true,
+                treat_err_as_bug: false,
+                external_macro_backtrace: false,
+                ..Default::default()
+            },
+        );
+
+        let mut sess = session::build_session_(
+            sessopts, cpath, diagnostic_handler, codemap,
+        );
+        let trans = rustc_driver::get_trans(&sess);
+        let cstore = Rc::new(CStore::new(trans.metadata_loader()));
+        rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
+
+        let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs));
+        target_features::add_configuration(&mut cfg, &sess, &*trans);
+        sess.parse_sess.config = cfg;
+
+        let control = &driver::CompileController::basic();
+
+        let krate = panictry!(driver::phase_1_parse_input(control, &sess, &input));
+
+        let name = ::rustc_trans_utils::link::find_crate_name(Some(&sess), &krate.attrs, &input);
+
+        let mut crate_loader = CrateLoader::new(&sess, &cstore, &name);
+
+        let resolver_arenas = resolve::Resolver::arenas();
+        let result = driver::phase_2_configure_and_expand_inner(&sess,
+                                                        &cstore,
+                                                        krate,
+                                                        None,
+                                                        &name,
+                                                        None,
+                                                        resolve::MakeGlobMap::No,
+                                                        &resolver_arenas,
+                                                        &mut crate_loader,
+                                                        |_| Ok(()));
+        let driver::InnerExpansionResult {
+            mut hir_forest,
+            resolver,
+            ..
+        } = abort_on_err(result, &sess);
+
+        // We need to hold on to the complete resolver, so we clone everything
+        // for the analysis passes to use. Suboptimal, but necessary in the
+        // current architecture.
+        let defs = resolver.definitions.clone();
+        let resolutions = ty::Resolutions {
+            freevars: resolver.freevars.clone(),
+            export_map: resolver.export_map.clone(),
+            trait_map: resolver.trait_map.clone(),
+            maybe_unused_trait_imports: resolver.maybe_unused_trait_imports.clone(),
+            maybe_unused_extern_crates: resolver.maybe_unused_extern_crates.clone(),
+        };
+        let analysis = ty::CrateAnalysis {
+            access_levels: Lrc::new(AccessLevels::default()),
+            name: name.to_string(),
+            glob_map: if resolver.make_glob_map { Some(resolver.glob_map.clone()) } else { None },
         };
 
-        let ctxt = DocContext {
+        let arenas = AllArenas::new();
-            tcx,
+        let hir_map = hir_map::map_crate(&sess, &*cstore, &mut hir_forest, &defs);
-            resolver: &resolver,
+        let output_filenames = driver::build_output_filenames(&input,
-            crate_name,
+                                                            &None,
-            cstore: cstore.clone(),
+                                                            &None,
-            populated_all_crate_impls: Cell::new(false),
+                                                            &[],
-            access_levels: RefCell::new(access_levels),
+                                                            &sess);
-            external_traits: Default::default(),
-            active_extern_traits: Default::default(),
-            renderinfo: Default::default(),
-            ty_substs: Default::default(),
-            lt_substs: Default::default(),
-            impl_trait_bounds: Default::default(),
-            mod_ids: Default::default(),
-            send_trait: send_trait,
-            fake_def_ids: RefCell::new(FxHashMap()),
-            all_fake_def_ids: RefCell::new(FxHashSet()),
-            generated_synthetics: RefCell::new(FxHashSet()),
-        };
-        debug!("crate: {:?}", tcx.hir.krate());
 
-        let krate = {
+        let resolver = RefCell::new(resolver);
-            let mut v = RustdocVisitor::new(&*cstore, &ctxt);
-            v.visit(tcx.hir.krate());
-            v.clean(&ctxt)
-        };
 
-        (krate, ctxt.renderinfo.into_inner())
+        abort_on_err(driver::phase_3_run_analysis_passes(&*trans,
-    }), &sess)
+                                                        control,
+                                                        &sess,
+                                                        &*cstore,
+                                                        hir_map,
+                                                        analysis,
+                                                        resolutions,
+                                                        &arenas,
+                                                        &name,
+                                                        &output_filenames,
+                                                        |tcx, analysis, _, result| {
+            if let Err(_) = result {
+                sess.fatal("Compilation failed, aborting rustdoc");
+            }
+
+            let ty::CrateAnalysis { access_levels, .. } = analysis;
+
+            // Convert from a NodeId set to a DefId set since we don't always have easy access
+            // to the map from defid -> nodeid
+            let access_levels = AccessLevels {
+                map: access_levels.map.iter()
+                                    .map(|(&k, &v)| (tcx.hir.local_def_id(k), v))
+                                    .collect()
+            };
+
+            let send_trait = if crate_name == Some("core".to_string()) {
+                clean::get_trait_def_id(&tcx, &["marker", "Send"], true)
+            } else {
+                clean::get_trait_def_id(&tcx, &["core", "marker", "Send"], false)
+            };
+
+            let ctxt = DocContext {
+                tcx,
+                resolver: &resolver,
+                crate_name,
+                cstore: cstore.clone(),
+                populated_all_crate_impls: Cell::new(false),
+                access_levels: RefCell::new(access_levels),
+                external_traits: Default::default(),
+                active_extern_traits: Default::default(),
+                renderinfo: Default::default(),
+                ty_substs: Default::default(),
+                lt_substs: Default::default(),
+                impl_trait_bounds: Default::default(),
+                mod_ids: Default::default(),
+                send_trait: send_trait,
+                fake_def_ids: RefCell::new(FxHashMap()),
+                all_fake_def_ids: RefCell::new(FxHashSet()),
+                generated_synthetics: RefCell::new(FxHashSet()),
+            };
+            debug!("crate: {:?}", tcx.hir.krate());
+
+            let krate = {
+                let mut v = RustdocVisitor::new(&*cstore, &ctxt);
+                v.visit(tcx.hir.krate());
+                v.clean(&ctxt)
+            };
+
+            (krate, ctxt.renderinfo.into_inner())
+        }), &sess)
+    })
 }

src/librustdoc/lib.rs

@@ -26,6 +26,8 @@
 #![feature(vec_remove_item)]
 #![feature(entry_and_modify)]
 
+#![recursion_limit="256"]
+
 extern crate arena;
 extern crate getopts;
 extern crate env_logger;

src/librustdoc/passes/propagate_doc_cfg.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::rc::Rc;
+use std::sync::Arc;
 
 use clean::{Crate, Item};
 use clean::cfg::Cfg;
@@ -20,7 +20,7 @@ pub fn propagate_doc_cfg(cr: Crate) -> PluginResult {
 }
 
 struct CfgPropagator {
-    parent_cfg: Option<Rc<Cfg>>,
+    parent_cfg: Option<Arc<Cfg>>,
 }
 
 impl DocFolder for CfgPropagator {
@@ -31,8 +31,8 @@ impl DocFolder for CfgPropagator {
             (None, None) => None,
             (Some(rc), None) | (None, Some(rc)) => Some(rc),
             (Some(mut a), Some(b)) => {
-                let b = Rc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
+                let b = Arc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
-                *Rc::make_mut(&mut a) &= b;
+                *Arc::make_mut(&mut a) &= b;
                 Some(a)
             }
         };

src/librustdoc/test.rs

@@ -85,77 +85,80 @@ pub fn run(input_path: &Path,
         edition,
         ..config::basic_options().clone()
     };
+    driver::spawn_thread_pool(sessopts, |sessopts| {
+        let codemap = Lrc::new(CodeMap::new(sessopts.file_path_mapping()));
+        let handler =
+            errors::Handler::with_tty_emitter(ColorConfig::Auto,
+                                            true, false,
+                                            Some(codemap.clone()));
 
-    let codemap = Lrc::new(CodeMap::new(sessopts.file_path_mapping()));
+        let mut sess = session::build_session_(
-    let handler =
+            sessopts, Some(input_path.to_owned()), handler, codemap.clone(),
-        errors::Handler::with_tty_emitter(ColorConfig::Auto,
+        );
-                                          true, false,
+        let trans = rustc_driver::get_trans(&sess);
-                                          Some(codemap.clone()));
+        let cstore = CStore::new(trans.metadata_loader());
+        rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
 
-    let mut sess = session::build_session_(
+        let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
-        sessopts, Some(input_path.to_owned()), handler, codemap.clone(),
+        target_features::add_configuration(&mut cfg, &sess, &*trans);
-    );
+        sess.parse_sess.config = cfg;
-    let trans = rustc_driver::get_trans(&sess);
-    let cstore = CStore::new(trans.metadata_loader());
-    rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
 
-    let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
+        let krate = panictry!(driver::phase_1_parse_input(&driver::CompileController::basic(),
-    target_features::add_configuration(&mut cfg, &sess, &*trans);
+                                                        &sess,
-    sess.parse_sess.config = cfg;
+                                                        &input));
-
+        let driver::ExpansionResult { defs, mut hir_forest, .. } = {
-    let krate = panictry!(driver::phase_1_parse_input(&driver::CompileController::basic(),
+            phase_2_configure_and_expand(
-                                                      &sess,
+                &sess,
-                                                      &input));
+                &cstore,
-    let driver::ExpansionResult { defs, mut hir_forest, .. } = {
+                krate,
-        phase_2_configure_and_expand(
+                None,
-            &sess,
+                "rustdoc-test",
-            &cstore,
+                None,
-            krate,
+                MakeGlobMap::No,
-            None,
+                |_| Ok(()),
-            "rustdoc-test",
+            ).expect("phase_2_configure_and_expand aborted in rustdoc!")
-            None,
-            MakeGlobMap::No,
-            |_| Ok(()),
-        ).expect("phase_2_configure_and_expand aborted in rustdoc!")
-    };
-
-    let crate_name = crate_name.unwrap_or_else(|| {
-        ::rustc_trans_utils::link::find_crate_name(None, &hir_forest.krate().attrs, &input)
-    });
-    let mut opts = scrape_test_config(hir_forest.krate());
-    opts.display_warnings |= display_warnings;
-    let mut collector = Collector::new(crate_name,
-                                       cfgs,
-                                       libs,
-                                       cg,
-                                       externs,
-                                       false,
-                                       opts,
-                                       maybe_sysroot,
-                                       Some(codemap),
-                                       None,
-                                       linker,
-                                       edition);
-
-    {
-        let map = hir::map::map_crate(&sess, &cstore, &mut hir_forest, &defs);
-        let krate = map.krate();
-        let mut hir_collector = HirCollector {
-            sess: &sess,
-            collector: &mut collector,
-            map: &map
         };
-        hir_collector.visit_testable("".to_string(), &krate.attrs, |this| {
+
-            intravisit::walk_crate(this, krate);
+        let crate_name = crate_name.unwrap_or_else(|| {
+            ::rustc_trans_utils::link::find_crate_name(None, &hir_forest.krate().attrs, &input)
         });
-    }
+        let mut opts = scrape_test_config(hir_forest.krate());
+        opts.display_warnings |= display_warnings;
+        let mut collector = Collector::new(
+            crate_name,
+            cfgs,
+            libs,
+            cg,
+            externs,
+            false,
+            opts,
+            maybe_sysroot,
+            Some(codemap),
+             None,
+            linker,
+            edition
+        );
 
-    test_args.insert(0, "rustdoctest".to_string());
+        {
+            let map = hir::map::map_crate(&sess, &cstore, &mut hir_forest, &defs);
+            let krate = map.krate();
+            let mut hir_collector = HirCollector {
+                sess: &sess,
+                collector: &mut collector,
+                map: &map
+            };
+            hir_collector.visit_testable("".to_string(), &krate.attrs, |this| {
+                intravisit::walk_crate(this, krate);
+            });
+        }
 
-    testing::test_main(&test_args,
+        test_args.insert(0, "rustdoctest".to_string());
-                       collector.tests.into_iter().collect(),
+
-                       testing::Options::new().display_output(display_warnings));
+        testing::test_main(&test_args,
-    0
+                        collector.tests.into_iter().collect(),
+                        testing::Options::new().display_output(display_warnings));
+        0
+    })
 }
 
 // Look for #![doc(test(no_crate_inject))], used by crates in the std facade
@@ -229,102 +232,106 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
         ..config::basic_options().clone()
     };
 
-    // Shuffle around a few input and output handles here. We're going to pass
+    let (libdir, outdir) = driver::spawn_thread_pool(sessopts, |sessopts| {
-    // an explicit handle into rustc to collect output messages, but we also
+        // Shuffle around a few input and output handles here. We're going to pass
-    // want to catch the error message that rustc prints when it fails.
+        // an explicit handle into rustc to collect output messages, but we also
-    //
+        // want to catch the error message that rustc prints when it fails.
-    // We take our thread-local stderr (likely set by the test runner) and replace
+        //
-    // it with a sink that is also passed to rustc itself. When this function
+        // We take our thread-local stderr (likely set by the test runner) and replace
-    // returns the output of the sink is copied onto the output of our own thread.
+        // it with a sink that is also passed to rustc itself. When this function
-    //
+        // returns the output of the sink is copied onto the output of our own thread.
-    // The basic idea is to not use a default Handler for rustc, and then also
+        //
-    // not print things by default to the actual stderr.
+        // The basic idea is to not use a default Handler for rustc, and then also
-    struct Sink(Arc<Mutex<Vec<u8>>>);
+        // not print things by default to the actual stderr.
-    impl Write for Sink {
+        struct Sink(Arc<Mutex<Vec<u8>>>);
-        fn write(&mut self, data: &[u8]) -> io::Result<usize> {
+        impl Write for Sink {
-            Write::write(&mut *self.0.lock().unwrap(), data)
+            fn write(&mut self, data: &[u8]) -> io::Result<usize> {
+                Write::write(&mut *self.0.lock().unwrap(), data)
+            }
+            fn flush(&mut self) -> io::Result<()> { Ok(()) }
         }
-        fn flush(&mut self) -> io::Result<()> { Ok(()) }
+        struct Bomb(Arc<Mutex<Vec<u8>>>, Box<Write+Send>);
-    }
+        impl Drop for Bomb {
-    struct Bomb(Arc<Mutex<Vec<u8>>>, Box<Write+Send>);
+            fn drop(&mut self) {
-    impl Drop for Bomb {
+                let _ = self.1.write_all(&self.0.lock().unwrap());
-        fn drop(&mut self) {
-            let _ = self.1.write_all(&self.0.lock().unwrap());
-        }
-    }
-    let data = Arc::new(Mutex::new(Vec::new()));
-    let codemap = Lrc::new(CodeMap::new_doctest(
-        sessopts.file_path_mapping(), filename.clone(), line as isize - line_offset as isize
-    ));
-    let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()),
-                                                      Some(codemap.clone()),
-                                                      false,
-                                                      false);
-    let old = io::set_panic(Some(box Sink(data.clone())));
-    let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout()));
-
-    // Compile the code
-    let diagnostic_handler = errors::Handler::with_emitter(true, false, box emitter);
-
-    let mut sess = session::build_session_(
-        sessopts, None, diagnostic_handler, codemap,
-    );
-    let trans = rustc_driver::get_trans(&sess);
-    let cstore = CStore::new(trans.metadata_loader());
-    rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
-
-    let outdir = Mutex::new(TempDir::new("rustdoctest").ok().expect("rustdoc needs a tempdir"));
-    let libdir = sess.target_filesearch(PathKind::All).get_lib_path();
-    let mut control = driver::CompileController::basic();
-
-    let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
-    target_features::add_configuration(&mut cfg, &sess, &*trans);
-    sess.parse_sess.config = cfg;
-
-    let out = Some(outdir.lock().unwrap().path().to_path_buf());
-
-    if no_run {
-        control.after_analysis.stop = Compilation::Stop;
-    }
-
-    let res = panic::catch_unwind(AssertUnwindSafe(|| {
-        driver::compile_input(
-            trans,
-            &sess,
-            &cstore,
-            &None,
-            &input,
-            &out,
-            &None,
-            None,
-            &control
-        )
-    }));
-
-    let compile_result = match res {
-        Ok(Ok(())) | Ok(Err(CompileIncomplete::Stopped)) => Ok(()),
-        Err(_) | Ok(Err(CompileIncomplete::Errored(_))) => Err(())
-    };
-
-    match (compile_result, compile_fail) {
-        (Ok(()), true) => {
-            panic!("test compiled while it wasn't supposed to")
-        }
-        (Ok(()), false) => {}
-        (Err(()), true) => {
-            if error_codes.len() > 0 {
-                let out = String::from_utf8(data.lock().unwrap().to_vec()).unwrap();
-                error_codes.retain(|err| !out.contains(err));
             }
         }
-        (Err(()), false) => {
+        let data = Arc::new(Mutex::new(Vec::new()));
-            panic!("couldn't compile the test")
+        let codemap = Lrc::new(CodeMap::new_doctest(
-        }
+            sessopts.file_path_mapping(), filename.clone(), line as isize - line_offset as isize
-    }
+        ));
+        let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()),
+                                                        Some(codemap.clone()),
+                                                        false,
+                                                        false);
+        let old = io::set_panic(Some(box Sink(data.clone())));
+        let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout()));
 
-    if error_codes.len() > 0 {
+        // Compile the code
-        panic!("Some expected error codes were not found: {:?}", error_codes);
+        let diagnostic_handler = errors::Handler::with_emitter(true, false, box emitter);
-    }
+
+        let mut sess = session::build_session_(
+            sessopts, None, diagnostic_handler, codemap,
+        );
+        let trans = rustc_driver::get_trans(&sess);
+        let cstore = CStore::new(trans.metadata_loader());
+        rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
+
+        let outdir = Mutex::new(TempDir::new("rustdoctest").ok().expect("rustdoc needs a tempdir"));
+        let libdir = sess.target_filesearch(PathKind::All).get_lib_path();
+        let mut control = driver::CompileController::basic();
+
+        let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
+        target_features::add_configuration(&mut cfg, &sess, &*trans);
+        sess.parse_sess.config = cfg;
+
+        let out = Some(outdir.lock().unwrap().path().to_path_buf());
+
+        if no_run {
+            control.after_analysis.stop = Compilation::Stop;
+        }
+
+        let res = panic::catch_unwind(AssertUnwindSafe(|| {
+            driver::compile_input(
+                trans,
+                &sess,
+                &cstore,
+                &None,
+                &input,
+                &out,
+                &None,
+                None,
+                &control
+            )
+        }));
+
+        let compile_result = match res {
+            Ok(Ok(())) | Ok(Err(CompileIncomplete::Stopped)) => Ok(()),
+            Err(_) | Ok(Err(CompileIncomplete::Errored(_))) => Err(())
+        };
+
+        match (compile_result, compile_fail) {
+            (Ok(()), true) => {
+                panic!("test compiled while it wasn't supposed to")
+            }
+            (Ok(()), false) => {}
+            (Err(()), true) => {
+                if error_codes.len() > 0 {
+                    let out = String::from_utf8(data.lock().unwrap().to_vec()).unwrap();
+                    error_codes.retain(|err| !out.contains(err));
+                }
+            }
+            (Err(()), false) => {
+                panic!("couldn't compile the test")
+            }
+        }
+
+        if error_codes.len() > 0 {
+            panic!("Some expected error codes were not found: {:?}", error_codes);
+        }
+
+        (libdir, outdir)
+    });
 
     if no_run { return }
 

src/libsyntax/lib.rs

@@ -73,7 +73,7 @@ macro_rules! unwrap_or {
     }
 }
 
-struct Globals {
+pub struct Globals {
     used_attrs: Lock<Vec<u64>>,
     known_attrs: Lock<Vec<u64>>,
     syntax_pos_globals: syntax_pos::Globals,
@@ -98,7 +98,7 @@ pub fn with_globals<F, R>(f: F) -> R
     })
 }
 
-scoped_thread_local!(static GLOBALS: Globals);
+scoped_thread_local!(pub static GLOBALS: Globals);
 
 #[macro_use]
 pub mod diagnostics {

src/tools/tidy/src/deps.rs

@@ -59,6 +59,7 @@ static WHITELIST_CRATES: &'static [CrateVersion] = &[
 static WHITELIST: &'static [Crate] = &[
     Crate("aho-corasick"),
     Crate("ar"),
+    Crate("arrayvec"),
     Crate("atty"),
     Crate("backtrace"),
     Crate("backtrace-sys"),
@@ -67,6 +68,10 @@ static WHITELIST: &'static [Crate] = &[
     Crate("cc"),
     Crate("cfg-if"),
     Crate("cmake"),
+    Crate("crossbeam-deque"),
+    Crate("crossbeam-epoch"),
+    Crate("crossbeam-utils"),
+    Crate("either"),
     Crate("ena"),
     Crate("env_logger"),
     Crate("filetime"),
@@ -82,7 +87,9 @@ static WHITELIST: &'static [Crate] = &[
     Crate("log"),
     Crate("log_settings"),
     Crate("memchr"),
+    Crate("memoffset"),
     Crate("miniz-sys"),
+    Crate("nodrop"),
     Crate("num_cpus"),
     Crate("owning_ref"),
     Crate("parking_lot"),
@@ -95,7 +102,10 @@ static WHITELIST: &'static [Crate] = &[
     Crate("regex-syntax"),
     Crate("remove_dir_all"),
     Crate("rustc-demangle"),
+    Crate("rustc-rayon"),
+    Crate("rustc-rayon-core"),
     Crate("scoped-tls"),
+    Crate("scopeguard"),
     Crate("smallvec"),
     Crate("stable_deref_trait"),
     Crate("tempdir"),