bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Add a Rayon thread pool

Browse source
This commit is contained in:
John Kåre Alsaker 2018-04-26 00:49:52 +02:00
parent 3df199680a
commit 022dff47e3
13 changed files with 430 additions and 344 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/librustc/session/mod.rs
View file

@ -869,10 +869,16 @@ impl Session {
ret
}
/// Returns the number of query threads that should be used for this
/// compilation
pub fn query_threads_from_opts(opts: &config::Options) -> usize {
opts.debugging_opts.query_threads.unwrap_or(1)
}
/// Returns the number of query threads that should be used for this
/// compilation
pub fn query_threads(&self) -> usize {
self.opts.debugging_opts.query_threads.unwrap_or(1)
Self::query_threads_from_opts(&self.opts)
}
/// Returns the number of codegen units that should be used for this

6
src/librustc/ty/context.rs
View file

@ -1800,10 +1800,12 @@ pub mod tls {
/// in librustc otherwise. It is used to when diagnostic messages are
/// emitted and stores them in the current query, if there is one.
fn track_diagnostic(diagnostic: &Diagnostic) {
with_context(|context| {
if let Some(ref query) = context.query {
with_context_opt(|icx| {
if let Some(icx) = icx {
if let Some(ref query) = icx.query {
query.diagnostics.lock().push(diagnostic.clone());
}
}
})
}

2
src/librustc_driver/Cargo.toml
View file

@ -13,6 +13,8 @@ arena = { path = "../libarena" }
graphviz = { path = "../libgraphviz" }
log = "0.4"
env_logger = { version = "0.5", default-features = false }
rustc-rayon = "0.1.0"
scoped-tls = { version = "0.1.1", features = ["nightly"] }
rustc = { path = "../librustc" }
rustc_allocator = { path = "../librustc_allocator" }
rustc_target = { path = "../librustc_target" }

47
src/librustc_driver/driver.rs
View file

@ -49,7 +49,7 @@ use std::fs;
use std::io::{self, Write};
use std::iter;
use std::path::{Path, PathBuf};
use rustc_data_structures::sync::Lrc;
use rustc_data_structures::sync::{self, Lrc};
use std::sync::mpsc;
use syntax::{self, ast, attr, diagnostics, visit};
use syntax::ext::base::ExtCtxt;
@ -64,6 +64,51 @@ use pretty::ReplaceBodyWithLoop;
use profile;
#[cfg(not(parallel_queries))]
pub fn spawn_thread_pool<F: FnOnce(config::Options) -> R + sync::Send, R: sync::Send>(
opts: config::Options,
f: F
) -> R {
f(opts)
}
#[cfg(parallel_queries)]
pub fn spawn_thread_pool<F: FnOnce(config::Options) -> R + sync::Send, R: sync::Send>(
opts: config::Options,
f: F
) -> R {
use syntax;
use syntax_pos;
use rayon::{ThreadPoolBuilder, ThreadPool};
let config = ThreadPoolBuilder::new().num_threads(Session::query_threads_from_opts(&opts))
.stack_size(16 * 1024 * 1024);
let with_pool = move |pool: &ThreadPool| {
pool.install(move || f(opts))
};
syntax::GLOBALS.with(|syntax_globals| {
syntax_pos::GLOBALS.with(|syntax_pos_globals| {
// The main handler run for each Rayon worker thread and sets up
// the thread local rustc uses. syntax_globals and syntax_pos_globals are
// captured and set on the new threads. ty::tls::with_thread_locals sets up
// thread local callbacks from libsyntax
let main_handler = move |worker: &mut FnMut()| {
syntax::GLOBALS.set(syntax_globals, || {
syntax_pos::GLOBALS.set(syntax_pos_globals, || {
ty::tls::with_thread_locals(|| {
worker()
})
})
})
};
ThreadPool::scoped_pool(config, main_handler, with_pool).unwrap()
})
})
}
pub fn compile_input(
trans: Box<TransCrate>,
sess: &Session,

44
src/librustc_driver/lib.rs
View file

@ -35,6 +35,7 @@ extern crate graphviz;
extern crate env_logger;
#[cfg(unix)]
extern crate libc;
extern crate rustc_rayon as rayon;
extern crate rustc;
extern crate rustc_allocator;
extern crate rustc_target;
@ -53,6 +54,7 @@ extern crate rustc_save_analysis;
extern crate rustc_traits;
extern crate rustc_trans_utils;
extern crate rustc_typeck;
extern crate scoped_tls;
extern crate serialize;
#[macro_use]
extern crate log;
@ -66,7 +68,7 @@ use pretty::{PpMode, UserIdentifiedItem};
use rustc_resolve as resolve;
use rustc_save_analysis as save;
use rustc_save_analysis::DumpHandler;
use rustc_data_structures::sync::Lrc;
use rustc_data_structures::sync::{self, Lrc};
use rustc_data_structures::OnDrop;
use rustc::session::{self, config, Session, build_session, CompileResult};
use rustc::session::CompileIncomplete;
@ -450,29 +452,12 @@ fn get_trans_sysroot(backend_name: &str) -> fn() -> Box<TransCrate> {
// See comments on CompilerCalls below for details about the callbacks argument.
// The FileLoader provides a way to load files from sources other than the file system.
pub fn run_compiler<'a>(args: &[String],
callbacks: &mut CompilerCalls<'a>,
callbacks: &mut (CompilerCalls<'a> + sync::Send),
file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
emitter_dest: Option<Box<Write + Send>>)
-> (CompileResult, Option<Session>)
{
syntax::with_globals(|| {
run_compiler_impl(args, callbacks, file_loader, emitter_dest)
})
}
fn run_compiler_impl<'a>(args: &[String],
callbacks: &mut CompilerCalls<'a>,
file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
emitter_dest: Option<Box<Write + Send>>)
-> (CompileResult, Option<Session>)
{
macro_rules! do_or_return {($expr: expr, $sess: expr) => {
match $expr {
Compilation::Stop => return (Ok(()), $sess),
Compilation::Continue => {}
}
}}
let matches = match handle_options(args) {
Some(matches) => matches,
None => return (Ok(()), None),
@ -480,6 +465,27 @@ fn run_compiler_impl<'a>(args: &[String],
let (sopts, cfg) = config::build_session_options_and_crate_config(&matches);
driver::spawn_thread_pool(sopts, |sopts| {
run_compiler_with_pool(matches, sopts, cfg, callbacks, file_loader, emitter_dest)
})
})
}
fn run_compiler_with_pool<'a>(
matches: getopts::Matches,
sopts: config::Options,
cfg: ast::CrateConfig,
callbacks: &mut (CompilerCalls<'a> + sync::Send),
file_loader: Option<Box<FileLoader + Send + Sync + 'static>>,
emitter_dest: Option<Box<Write + Send>>
) -> (CompileResult, Option<Session>) {
macro_rules! do_or_return {($expr: expr, $sess: expr) => {
match $expr {
Compilation::Stop => return (Ok(()), $sess),
Compilation::Continue => {}
}
}}
let descriptions = diagnostics_registry();
do_or_return!(callbacks.early_callback(&matches,

25
src/librustc_driver/test.rs
View file

@ -99,20 +99,25 @@ fn test_env<F>(source_string: &str,
where F: FnOnce(Env)
{
syntax::with_globals(|| {
test_env_impl(source_string, args, body)
});
}
fn test_env_impl<F>(source_string: &str,
(emitter, expected_err_count): (Box<Emitter + sync::Send>, usize),
body: F)
where F: FnOnce(Env)
{
let mut options = config::basic_options();
options.debugging_opts.verbose = true;
options.unstable_features = UnstableFeatures::Allow;
let diagnostic_handler = errors::Handler::with_emitter(true, false, emitter);
driver::spawn_thread_pool(options, |options| {
test_env_with_pool(options, source_string, args, body)
})
});
}
fn test_env_with_pool<F>(
options: config::Options,
source_string: &str,
(emitter, expected_err_count): (Box<Emitter + sync::Send>, usize),
body: F
)
where F: FnOnce(Env)
{
let diagnostic_handler = errors::Handler::with_emitter(true, false, emitter);
let sess = session::build_session_(options,
None,
diagnostic_handler,

4
src/librustdoc/clean/mod.rs
View file

@ -697,7 +697,7 @@ impl<'a> FromIterator<&'a DocFragment> for String {
pub struct Attributes {
pub doc_strings: Vec<DocFragment>,
pub other_attrs: Vec<ast::Attribute>,
pub cfg: Option<Rc<Cfg>>,
pub cfg: Option<Arc<Cfg>>,
pub span: Option<syntax_pos::Span>,
/// map from Rust paths to resolved defs and potential URL fragments
pub links: Vec<(String, Option<DefId>, Option<String>)>,
@ -848,7 +848,7 @@ impl Attributes {
Attributes {
doc_strings,
other_attrs,
cfg: if cfg == Cfg::True { None } else { Some(Rc::new(cfg)) },
cfg: if cfg == Cfg::True { None } else { Some(Arc::new(cfg)) },
span: sp,
links: vec![],
}

3
src/librustdoc/core.rs
View file

@ -161,7 +161,7 @@ pub fn run_core(search_paths: SearchPaths,
edition,
..config::basic_options().clone()
};
driver::spawn_thread_pool(sessopts, move |sessopts| {
let codemap = Lrc::new(codemap::CodeMap::new(sessopts.file_path_mapping()));
let emitter: Box<dyn Emitter + sync::Send> = match error_format {
ErrorOutputType::HumanReadable(color_config) => Box::new(
@ -318,4 +318,5 @@ pub fn run_core(search_paths: SearchPaths,
(krate, ctxt.renderinfo.into_inner())
}), &sess)
})
}

2
src/librustdoc/lib.rs
View file

@ -26,6 +26,8 @@
#![feature(vec_remove_item)]
#![feature(entry_and_modify)]
#![recursion_limit="256"]
extern crate arena;
extern crate getopts;
extern crate env_logger;

8
src/librustdoc/passes/propagate_doc_cfg.rs
View file

@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::rc::Rc;
use std::sync::Arc;
use clean::{Crate, Item};
use clean::cfg::Cfg;
@ -20,7 +20,7 @@ pub fn propagate_doc_cfg(cr: Crate) -> PluginResult {
}
struct CfgPropagator {
parent_cfg: Option<Rc<Cfg>>,
parent_cfg: Option<Arc<Cfg>>,
}
impl DocFolder for CfgPropagator {
@ -31,8 +31,8 @@ impl DocFolder for CfgPropagator {
(None, None) => None,
(Some(rc), None) | (None, Some(rc)) => Some(rc),
(Some(mut a), Some(b)) => {
let b = Rc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
*Rc::make_mut(&mut a) &= b;
let b = Arc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
*Arc::make_mut(&mut a) &= b;
Some(a)
}
};

13
src/librustdoc/test.rs
View file

@ -85,7 +85,7 @@ pub fn run(input_path: &Path,
edition,
..config::basic_options().clone()
};
driver::spawn_thread_pool(sessopts, |sessopts| {
let codemap = Lrc::new(CodeMap::new(sessopts.file_path_mapping()));
let handler =
errors::Handler::with_tty_emitter(ColorConfig::Auto,
@ -124,7 +124,8 @@ pub fn run(input_path: &Path,
});
let mut opts = scrape_test_config(hir_forest.krate());
opts.display_warnings |= display_warnings;
let mut collector = Collector::new(crate_name,
let mut collector = Collector::new(
crate_name,
cfgs,
libs,
cg,
@ -135,7 +136,8 @@ pub fn run(input_path: &Path,
Some(codemap),
None,
linker,
edition);
edition
);
{
let map = hir::map::map_crate(&sess, &cstore, &mut hir_forest, &defs);
@ -156,6 +158,7 @@ pub fn run(input_path: &Path,
collector.tests.into_iter().collect(),
testing::Options::new().display_output(display_warnings));
0
})
}
// Look for #![doc(test(no_crate_inject))], used by crates in the std facade
@ -229,6 +232,7 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
..config::basic_options().clone()
};
let (libdir, outdir) = driver::spawn_thread_pool(sessopts, |sessopts| {
// Shuffle around a few input and output handles here. We're going to pass
// an explicit handle into rustc to collect output messages, but we also
// want to catch the error message that rustc prints when it fails.
@ -326,6 +330,9 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
panic!("Some expected error codes were not found: {:?}", error_codes);
}
(libdir, outdir)
});
if no_run { return }
// Run the code!

4
src/libsyntax/lib.rs
View file

@ -73,7 +73,7 @@ macro_rules! unwrap_or {
}
}
struct Globals {
pub struct Globals {
used_attrs: Lock<Vec<u64>>,
known_attrs: Lock<Vec<u64>>,
syntax_pos_globals: syntax_pos::Globals,
@ -98,7 +98,7 @@ pub fn with_globals<F, R>(f: F) -> R
})
}
scoped_thread_local!(static GLOBALS: Globals);
scoped_thread_local!(pub static GLOBALS: Globals);
#[macro_use]
pub mod diagnostics {

10
src/tools/tidy/src/deps.rs
View file

@ -59,6 +59,7 @@ static WHITELIST_CRATES: &'static [CrateVersion] = &[
static WHITELIST: &'static [Crate] = &[
Crate("aho-corasick"),
Crate("ar"),
Crate("arrayvec"),
Crate("atty"),
Crate("backtrace"),
Crate("backtrace-sys"),
@ -67,6 +68,10 @@ static WHITELIST: &'static [Crate] = &[
Crate("cc"),
Crate("cfg-if"),
Crate("cmake"),
Crate("crossbeam-deque"),
Crate("crossbeam-epoch"),
Crate("crossbeam-utils"),
Crate("either"),
Crate("ena"),
Crate("env_logger"),
Crate("filetime"),
@ -82,7 +87,9 @@ static WHITELIST: &'static [Crate] = &[
Crate("log"),
Crate("log_settings"),
Crate("memchr"),
Crate("memoffset"),
Crate("miniz-sys"),
Crate("nodrop"),
Crate("num_cpus"),
Crate("owning_ref"),
Crate("parking_lot"),
@ -95,7 +102,10 @@ static WHITELIST: &'static [Crate] = &[
Crate("regex-syntax"),
Crate("remove_dir_all"),
Crate("rustc-demangle"),
Crate("rustc-rayon"),
Crate("rustc-rayon-core"),
Crate("scoped-tls"),
Crate("scopeguard"),
Crate("smallvec"),
Crate("stable_deref_trait"),
Crate("tempdir"),