Add cargo lintcheck --recursive
to check dependencies of crates
This commit is contained in:
parent
826a8930e6
commit
fc77d91469
7 changed files with 339 additions and 55 deletions
|
@ -12,9 +12,11 @@ publish = false
|
||||||
[dependencies]
|
[dependencies]
|
||||||
cargo_metadata = "0.14"
|
cargo_metadata = "0.14"
|
||||||
clap = "3.2"
|
clap = "3.2"
|
||||||
|
crossbeam-channel = "0.5.6"
|
||||||
flate2 = "1.0"
|
flate2 = "1.0"
|
||||||
rayon = "1.5.1"
|
rayon = "1.5.1"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0.85"
|
||||||
tar = "0.4"
|
tar = "0.4"
|
||||||
toml = "0.5"
|
toml = "0.5"
|
||||||
ureq = "2.2"
|
ureq = "2.2"
|
||||||
|
|
|
@ -69,9 +69,27 @@ is checked.
|
||||||
is explicitly specified in the options.
|
is explicitly specified in the options.
|
||||||
|
|
||||||
### Fix mode
|
### Fix mode
|
||||||
You can run `./lintcheck/target/debug/lintcheck --fix` which will run Clippy with `--fix` and
|
You can run `cargo lintcheck --fix` which will run Clippy with `--fix` and
|
||||||
print a warning if Clippy's suggestions fail to apply (if the resulting code does not build).
|
print a warning if Clippy's suggestions fail to apply (if the resulting code does not build).
|
||||||
This lets us spot bad suggestions or false positives automatically in some cases.
|
This lets us spot bad suggestions or false positives automatically in some cases.
|
||||||
|
|
||||||
Please note that the target dir should be cleaned afterwards since clippy will modify
|
Please note that the target dir should be cleaned afterwards since clippy will modify
|
||||||
the downloaded sources which can lead to unexpected results when running lintcheck again afterwards.
|
the downloaded sources which can lead to unexpected results when running lintcheck again afterwards.
|
||||||
|
|
||||||
|
### Recursive mode
|
||||||
|
You can run `cargo lintcheck --recursive` to also run Clippy on the dependencies
|
||||||
|
of the crates listed in the crates source `.toml`. e.g. adding `rand 0.8.5`
|
||||||
|
would also lint `rand_core`, `rand_chacha`, etc.
|
||||||
|
|
||||||
|
Particularly slow crates in the dependency graph can be ignored using
|
||||||
|
`recursive.ignore`:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[crates]
|
||||||
|
cargo = {name = "cargo", versions = ['0.64.0']}
|
||||||
|
|
||||||
|
[recursive]
|
||||||
|
ignore = [
|
||||||
|
"unicode-normalization",
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
|
@ -33,3 +33,11 @@ cfg-expr = {name = "cfg-expr", versions = ['0.7.1']}
|
||||||
puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
|
puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
|
||||||
rpmalloc = {name = "rpmalloc", versions = ['0.2.0']}
|
rpmalloc = {name = "rpmalloc", versions = ['0.2.0']}
|
||||||
tame-oidc = {name = "tame-oidc", versions = ['0.1.0']}
|
tame-oidc = {name = "tame-oidc", versions = ['0.1.0']}
|
||||||
|
|
||||||
|
[recursive]
|
||||||
|
ignore = [
|
||||||
|
# Takes ~30s to lint
|
||||||
|
"combine",
|
||||||
|
# Has 1.2 million `clippy::match_same_arms`s
|
||||||
|
"unicode-normalization",
|
||||||
|
]
|
||||||
|
|
|
@ -34,11 +34,16 @@ fn get_clap_config() -> ArgMatches {
|
||||||
Arg::new("markdown")
|
Arg::new("markdown")
|
||||||
.long("markdown")
|
.long("markdown")
|
||||||
.help("Change the reports table to use markdown links"),
|
.help("Change the reports table to use markdown links"),
|
||||||
|
Arg::new("recursive")
|
||||||
|
.long("--recursive")
|
||||||
|
.help("Run clippy on the dependencies of crates specified in crates-toml")
|
||||||
|
.conflicts_with("threads")
|
||||||
|
.conflicts_with("fix"),
|
||||||
])
|
])
|
||||||
.get_matches()
|
.get_matches()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct LintcheckConfig {
|
pub(crate) struct LintcheckConfig {
|
||||||
/// max number of jobs to spawn (default 1)
|
/// max number of jobs to spawn (default 1)
|
||||||
pub max_jobs: usize,
|
pub max_jobs: usize,
|
||||||
|
@ -54,6 +59,8 @@ pub(crate) struct LintcheckConfig {
|
||||||
pub lint_filter: Vec<String>,
|
pub lint_filter: Vec<String>,
|
||||||
/// Indicate if the output should support markdown syntax
|
/// Indicate if the output should support markdown syntax
|
||||||
pub markdown: bool,
|
pub markdown: bool,
|
||||||
|
/// Run clippy on the dependencies of crates
|
||||||
|
pub recursive: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LintcheckConfig {
|
impl LintcheckConfig {
|
||||||
|
@ -119,6 +126,7 @@ impl LintcheckConfig {
|
||||||
fix: clap_config.contains_id("fix"),
|
fix: clap_config.contains_id("fix"),
|
||||||
lint_filter,
|
lint_filter,
|
||||||
markdown,
|
markdown,
|
||||||
|
recursive: clap_config.contains_id("recursive"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
67
lintcheck/src/driver.rs
Normal file
67
lintcheck/src/driver.rs
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
use crate::recursive::{deserialize_line, serialize_line, DriverInfo};
|
||||||
|
|
||||||
|
use std::io::{self, BufReader, Write};
|
||||||
|
use std::net::TcpStream;
|
||||||
|
use std::process::{self, Command, Stdio};
|
||||||
|
use std::{env, mem};
|
||||||
|
|
||||||
|
/// 1. Sends [DriverInfo] to the [crate::recursive::LintcheckServer] running on `addr`
|
||||||
|
/// 2. Receives [bool] from the server, if `false` returns `None`
|
||||||
|
/// 3. Otherwise sends the stderr of running `clippy-driver` to the server
|
||||||
|
fn run_clippy(addr: &str) -> Option<i32> {
|
||||||
|
let driver_info = DriverInfo {
|
||||||
|
package_name: env::var("CARGO_PKG_NAME").ok()?,
|
||||||
|
crate_name: env::var("CARGO_CRATE_NAME").ok()?,
|
||||||
|
version: env::var("CARGO_PKG_VERSION").ok()?,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut stream = BufReader::new(TcpStream::connect(addr).unwrap());
|
||||||
|
|
||||||
|
serialize_line(&driver_info, stream.get_mut());
|
||||||
|
|
||||||
|
let should_run = deserialize_line::<bool, _>(&mut stream);
|
||||||
|
if !should_run {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove --cap-lints allow so that clippy runs and lints are emitted
|
||||||
|
let mut include_next = true;
|
||||||
|
let args = env::args().skip(1).filter(|arg| match arg.as_str() {
|
||||||
|
"--cap-lints=allow" => false,
|
||||||
|
"--cap-lints" => {
|
||||||
|
include_next = false;
|
||||||
|
false
|
||||||
|
},
|
||||||
|
_ => mem::replace(&mut include_next, true),
|
||||||
|
});
|
||||||
|
|
||||||
|
let output = Command::new(env::var("CLIPPY_DRIVER").expect("missing env CLIPPY_DRIVER"))
|
||||||
|
.args(args)
|
||||||
|
.stdout(Stdio::inherit())
|
||||||
|
.output()
|
||||||
|
.expect("failed to run clippy-driver");
|
||||||
|
|
||||||
|
stream
|
||||||
|
.get_mut()
|
||||||
|
.write_all(&output.stderr)
|
||||||
|
.unwrap_or_else(|e| panic!("{e:?} in {driver_info:?}"));
|
||||||
|
|
||||||
|
match output.status.code() {
|
||||||
|
Some(0) => Some(0),
|
||||||
|
code => {
|
||||||
|
io::stderr().write_all(&output.stderr).unwrap();
|
||||||
|
Some(code.expect("killed by signal"))
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn drive(addr: &str) {
|
||||||
|
process::exit(run_clippy(addr).unwrap_or_else(|| {
|
||||||
|
Command::new("rustc")
|
||||||
|
.args(env::args_os().skip(2))
|
||||||
|
.status()
|
||||||
|
.unwrap()
|
||||||
|
.code()
|
||||||
|
.unwrap()
|
||||||
|
}))
|
||||||
|
}
|
|
@ -8,13 +8,17 @@
|
||||||
#![allow(clippy::collapsible_else_if)]
|
#![allow(clippy::collapsible_else_if)]
|
||||||
|
|
||||||
mod config;
|
mod config;
|
||||||
|
mod driver;
|
||||||
|
mod recursive;
|
||||||
|
|
||||||
use config::LintcheckConfig;
|
use crate::config::LintcheckConfig;
|
||||||
|
use crate::recursive::LintcheckServer;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::env;
|
use std::env;
|
||||||
|
use std::env::consts::EXE_SUFFIX;
|
||||||
use std::fmt::Write as _;
|
use std::fmt::Write as _;
|
||||||
use std::fs::write;
|
use std::fs;
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
@ -22,22 +26,12 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use cargo_metadata::diagnostic::DiagnosticLevel;
|
use cargo_metadata::diagnostic::{Diagnostic, DiagnosticLevel};
|
||||||
use cargo_metadata::Message;
|
use cargo_metadata::Message;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use walkdir::{DirEntry, WalkDir};
|
use walkdir::{DirEntry, WalkDir};
|
||||||
|
|
||||||
#[cfg(not(windows))]
|
|
||||||
const CLIPPY_DRIVER_PATH: &str = "target/debug/clippy-driver";
|
|
||||||
#[cfg(not(windows))]
|
|
||||||
const CARGO_CLIPPY_PATH: &str = "target/debug/cargo-clippy";
|
|
||||||
|
|
||||||
#[cfg(windows)]
|
|
||||||
const CLIPPY_DRIVER_PATH: &str = "target/debug/clippy-driver.exe";
|
|
||||||
#[cfg(windows)]
|
|
||||||
const CARGO_CLIPPY_PATH: &str = "target/debug/cargo-clippy.exe";
|
|
||||||
|
|
||||||
const LINTCHECK_DOWNLOADS: &str = "target/lintcheck/downloads";
|
const LINTCHECK_DOWNLOADS: &str = "target/lintcheck/downloads";
|
||||||
const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";
|
const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";
|
||||||
|
|
||||||
|
@ -45,6 +39,13 @@ const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct SourceList {
|
struct SourceList {
|
||||||
crates: HashMap<String, TomlCrate>,
|
crates: HashMap<String, TomlCrate>,
|
||||||
|
#[serde(default)]
|
||||||
|
recursive: RecursiveOptions,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||||
|
struct RecursiveOptions {
|
||||||
|
ignore: HashSet<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A crate source stored inside the .toml
|
/// A crate source stored inside the .toml
|
||||||
|
@ -105,12 +106,7 @@ struct ClippyWarning {
|
||||||
|
|
||||||
#[allow(unused)]
|
#[allow(unused)]
|
||||||
impl ClippyWarning {
|
impl ClippyWarning {
|
||||||
fn new(cargo_message: Message, krate: &Crate) -> Option<Self> {
|
fn new(diag: Diagnostic, crate_name: &str, crate_version: &str) -> Option<Self> {
|
||||||
let diag = match cargo_message {
|
|
||||||
Message::CompilerMessage(message) => message.message,
|
|
||||||
_ => return None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let lint_type = diag.code?.code;
|
let lint_type = diag.code?.code;
|
||||||
if !(lint_type.contains("clippy") || diag.message.contains("clippy"))
|
if !(lint_type.contains("clippy") || diag.message.contains("clippy"))
|
||||||
|| diag.message.contains("could not read cargo metadata")
|
|| diag.message.contains("could not read cargo metadata")
|
||||||
|
@ -124,12 +120,12 @@ impl ClippyWarning {
|
||||||
Ok(stripped) => format!("$CARGO_HOME/{}", stripped.display()),
|
Ok(stripped) => format!("$CARGO_HOME/{}", stripped.display()),
|
||||||
Err(_) => format!(
|
Err(_) => format!(
|
||||||
"target/lintcheck/sources/{}-{}/{}",
|
"target/lintcheck/sources/{}-{}/{}",
|
||||||
krate.name, krate.version, span.file_name
|
crate_name, crate_version, span.file_name
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(Self {
|
Some(Self {
|
||||||
crate_name: krate.name.clone(),
|
crate_name: crate_name.to_owned(),
|
||||||
file,
|
file,
|
||||||
line: span.line_start,
|
line: span.line_start,
|
||||||
column: span.column_start,
|
column: span.column_start,
|
||||||
|
@ -142,8 +138,6 @@ impl ClippyWarning {
|
||||||
fn to_output(&self, markdown: bool) -> String {
|
fn to_output(&self, markdown: bool) -> String {
|
||||||
let file_with_pos = format!("{}:{}:{}", &self.file, &self.line, &self.column);
|
let file_with_pos = format!("{}:{}:{}", &self.file, &self.line, &self.column);
|
||||||
if markdown {
|
if markdown {
|
||||||
let lint = format!("`{}`", self.lint_type);
|
|
||||||
|
|
||||||
let mut file = self.file.clone();
|
let mut file = self.file.clone();
|
||||||
if !file.starts_with('$') {
|
if !file.starts_with('$') {
|
||||||
file.insert_str(0, "../");
|
file.insert_str(0, "../");
|
||||||
|
@ -151,7 +145,7 @@ impl ClippyWarning {
|
||||||
|
|
||||||
let mut output = String::from("| ");
|
let mut output = String::from("| ");
|
||||||
let _ = write!(output, "[`{}`]({}#L{})", file_with_pos, file, self.line);
|
let _ = write!(output, "[`{}`]({}#L{})", file_with_pos, file, self.line);
|
||||||
let _ = write!(output, r#" | {:<50} | "{}" |"#, lint, self.message);
|
let _ = write!(output, r#" | `{:<50}` | "{}" |"#, self.lint_type, self.message);
|
||||||
output.push('\n');
|
output.push('\n');
|
||||||
output
|
output
|
||||||
} else {
|
} else {
|
||||||
|
@ -243,6 +237,7 @@ impl CrateSource {
|
||||||
}
|
}
|
||||||
// check out the commit/branch/whatever
|
// check out the commit/branch/whatever
|
||||||
if !Command::new("git")
|
if !Command::new("git")
|
||||||
|
.args(["-c", "advice.detachedHead=false"])
|
||||||
.arg("checkout")
|
.arg("checkout")
|
||||||
.arg(commit)
|
.arg(commit)
|
||||||
.current_dir(&repo_path)
|
.current_dir(&repo_path)
|
||||||
|
@ -309,10 +304,12 @@ impl Crate {
|
||||||
fn run_clippy_lints(
|
fn run_clippy_lints(
|
||||||
&self,
|
&self,
|
||||||
cargo_clippy_path: &Path,
|
cargo_clippy_path: &Path,
|
||||||
|
clippy_driver_path: &Path,
|
||||||
target_dir_index: &AtomicUsize,
|
target_dir_index: &AtomicUsize,
|
||||||
total_crates_to_lint: usize,
|
total_crates_to_lint: usize,
|
||||||
config: &LintcheckConfig,
|
config: &LintcheckConfig,
|
||||||
lint_filter: &Vec<String>,
|
lint_filter: &Vec<String>,
|
||||||
|
server: &Option<LintcheckServer>,
|
||||||
) -> Vec<ClippyWarning> {
|
) -> Vec<ClippyWarning> {
|
||||||
// advance the atomic index by one
|
// advance the atomic index by one
|
||||||
let index = target_dir_index.fetch_add(1, Ordering::SeqCst);
|
let index = target_dir_index.fetch_add(1, Ordering::SeqCst);
|
||||||
|
@ -336,36 +333,67 @@ impl Crate {
|
||||||
|
|
||||||
let shared_target_dir = clippy_project_root().join("target/lintcheck/shared_target_dir");
|
let shared_target_dir = clippy_project_root().join("target/lintcheck/shared_target_dir");
|
||||||
|
|
||||||
let mut args = if config.fix {
|
let mut cargo_clippy_args = if config.fix {
|
||||||
vec!["--fix", "--"]
|
vec!["--fix", "--"]
|
||||||
} else {
|
} else {
|
||||||
vec!["--", "--message-format=json", "--"]
|
vec!["--", "--message-format=json", "--"]
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut clippy_args = Vec::<&str>::new();
|
||||||
if let Some(options) = &self.options {
|
if let Some(options) = &self.options {
|
||||||
for opt in options {
|
for opt in options {
|
||||||
args.push(opt);
|
clippy_args.push(opt);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
args.extend(&["-Wclippy::pedantic", "-Wclippy::cargo"])
|
clippy_args.extend(&["-Wclippy::pedantic", "-Wclippy::cargo"])
|
||||||
}
|
}
|
||||||
|
|
||||||
if lint_filter.is_empty() {
|
if lint_filter.is_empty() {
|
||||||
args.push("--cap-lints=warn");
|
clippy_args.push("--cap-lints=warn");
|
||||||
} else {
|
} else {
|
||||||
args.push("--cap-lints=allow");
|
clippy_args.push("--cap-lints=allow");
|
||||||
args.extend(lint_filter.iter().map(|filter| filter.as_str()))
|
clippy_args.extend(lint_filter.iter().map(|filter| filter.as_str()))
|
||||||
}
|
}
|
||||||
|
|
||||||
let all_output = std::process::Command::new(&cargo_clippy_path)
|
if let Some(server) = server {
|
||||||
|
let target = shared_target_dir.join("recursive");
|
||||||
|
|
||||||
|
// `cargo clippy` is a wrapper around `cargo check` that mainly sets `RUSTC_WORKSPACE_WRAPPER` to
|
||||||
|
// `clippy-driver`. We do the same thing here with a couple changes:
|
||||||
|
//
|
||||||
|
// `RUSTC_WRAPPER` is used instead of `RUSTC_WORKSPACE_WRAPPER` so that we can lint all crate
|
||||||
|
// dependencies rather than only workspace members
|
||||||
|
//
|
||||||
|
// The wrapper is set to the `lintcheck` so we can force enable linting and ignore certain crates
|
||||||
|
// (see `crate::driver`)
|
||||||
|
let status = Command::new("cargo")
|
||||||
|
.arg("check")
|
||||||
|
.arg("--quiet")
|
||||||
|
.current_dir(&self.path)
|
||||||
|
.env("CLIPPY_ARGS", clippy_args.join("__CLIPPY_HACKERY__"))
|
||||||
|
.env("CARGO_TARGET_DIR", target)
|
||||||
|
.env("RUSTC_WRAPPER", env::current_exe().unwrap())
|
||||||
|
// Pass the absolute path so `crate::driver` can find `clippy-driver`, as it's executed in various
|
||||||
|
// different working directories
|
||||||
|
.env("CLIPPY_DRIVER", clippy_driver_path)
|
||||||
|
.env("LINTCHECK_SERVER", server.local_addr.to_string())
|
||||||
|
.status()
|
||||||
|
.expect("failed to run cargo");
|
||||||
|
|
||||||
|
assert_eq!(status.code(), Some(0));
|
||||||
|
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
cargo_clippy_args.extend(clippy_args);
|
||||||
|
|
||||||
|
let all_output = Command::new(&cargo_clippy_path)
|
||||||
// use the looping index to create individual target dirs
|
// use the looping index to create individual target dirs
|
||||||
.env(
|
.env(
|
||||||
"CARGO_TARGET_DIR",
|
"CARGO_TARGET_DIR",
|
||||||
shared_target_dir.join(format!("_{:?}", thread_index)),
|
shared_target_dir.join(format!("_{:?}", thread_index)),
|
||||||
)
|
)
|
||||||
// lint warnings will look like this:
|
.args(&cargo_clippy_args)
|
||||||
// src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
|
|
||||||
.args(&args)
|
|
||||||
.current_dir(&self.path)
|
.current_dir(&self.path)
|
||||||
.output()
|
.output()
|
||||||
.unwrap_or_else(|error| {
|
.unwrap_or_else(|error| {
|
||||||
|
@ -404,7 +432,10 @@ impl Crate {
|
||||||
|
|
||||||
// get all clippy warnings and ICEs
|
// get all clippy warnings and ICEs
|
||||||
let warnings: Vec<ClippyWarning> = Message::parse_stream(stdout.as_bytes())
|
let warnings: Vec<ClippyWarning> = Message::parse_stream(stdout.as_bytes())
|
||||||
.filter_map(|msg| ClippyWarning::new(msg.unwrap(), &self))
|
.filter_map(|msg| match msg {
|
||||||
|
Ok(Message::CompilerMessage(message)) => ClippyWarning::new(message.message, &self.name, &self.version),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
warnings
|
warnings
|
||||||
|
@ -423,8 +454,8 @@ fn build_clippy() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read a `toml` file and return a list of `CrateSources` that we want to check with clippy
|
/// Read a `lintcheck_crates.toml` file
|
||||||
fn read_crates(toml_path: &Path) -> Vec<CrateSource> {
|
fn read_crates(toml_path: &Path) -> (Vec<CrateSource>, RecursiveOptions) {
|
||||||
let toml_content: String =
|
let toml_content: String =
|
||||||
std::fs::read_to_string(&toml_path).unwrap_or_else(|_| panic!("Failed to read {}", toml_path.display()));
|
std::fs::read_to_string(&toml_path).unwrap_or_else(|_| panic!("Failed to read {}", toml_path.display()));
|
||||||
let crate_list: SourceList =
|
let crate_list: SourceList =
|
||||||
|
@ -484,7 +515,7 @@ fn read_crates(toml_path: &Path) -> Vec<CrateSource> {
|
||||||
// sort the crates
|
// sort the crates
|
||||||
crate_sources.sort();
|
crate_sources.sort();
|
||||||
|
|
||||||
crate_sources
|
(crate_sources, crate_list.recursive)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate a short list of occurring lints-types and their count
|
/// Generate a short list of occurring lints-types and their count
|
||||||
|
@ -516,20 +547,20 @@ fn gather_stats(clippy_warnings: &[ClippyWarning]) -> (String, HashMap<&String,
|
||||||
|
|
||||||
/// check if the latest modification of the logfile is older than the modification date of the
|
/// check if the latest modification of the logfile is older than the modification date of the
|
||||||
/// clippy binary, if this is true, we should clean the lintchec shared target directory and recheck
|
/// clippy binary, if this is true, we should clean the lintchec shared target directory and recheck
|
||||||
fn lintcheck_needs_rerun(lintcheck_logs_path: &Path) -> bool {
|
fn lintcheck_needs_rerun(lintcheck_logs_path: &Path, paths: [&Path; 2]) -> bool {
|
||||||
if !lintcheck_logs_path.exists() {
|
if !lintcheck_logs_path.exists() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
let clippy_modified: std::time::SystemTime = {
|
let clippy_modified: std::time::SystemTime = {
|
||||||
let mut times = [CLIPPY_DRIVER_PATH, CARGO_CLIPPY_PATH].iter().map(|p| {
|
let [cargo, driver] = paths.map(|p| {
|
||||||
std::fs::metadata(p)
|
std::fs::metadata(p)
|
||||||
.expect("failed to get metadata of file")
|
.expect("failed to get metadata of file")
|
||||||
.modified()
|
.modified()
|
||||||
.expect("failed to get modification date")
|
.expect("failed to get modification date")
|
||||||
});
|
});
|
||||||
// the oldest modification of either of the binaries
|
// the oldest modification of either of the binaries
|
||||||
std::cmp::max(times.next().unwrap(), times.next().unwrap())
|
std::cmp::max(cargo, driver)
|
||||||
};
|
};
|
||||||
|
|
||||||
let logs_modified: std::time::SystemTime = std::fs::metadata(lintcheck_logs_path)
|
let logs_modified: std::time::SystemTime = std::fs::metadata(lintcheck_logs_path)
|
||||||
|
@ -543,6 +574,11 @@ fn lintcheck_needs_rerun(lintcheck_logs_path: &Path) -> bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
|
// We're being executed as a `RUSTC_WRAPPER` as part of `--recursive`
|
||||||
|
if let Ok(addr) = env::var("LINTCHECK_SERVER") {
|
||||||
|
driver::drive(&addr);
|
||||||
|
}
|
||||||
|
|
||||||
// assert that we launch lintcheck from the repo root (via cargo lintcheck)
|
// assert that we launch lintcheck from the repo root (via cargo lintcheck)
|
||||||
if std::fs::metadata("lintcheck/Cargo.toml").is_err() {
|
if std::fs::metadata("lintcheck/Cargo.toml").is_err() {
|
||||||
eprintln!("lintcheck needs to be run from clippy's repo root!\nUse `cargo lintcheck` alternatively.");
|
eprintln!("lintcheck needs to be run from clippy's repo root!\nUse `cargo lintcheck` alternatively.");
|
||||||
|
@ -555,9 +591,15 @@ fn main() {
|
||||||
build_clippy();
|
build_clippy();
|
||||||
println!("Done compiling");
|
println!("Done compiling");
|
||||||
|
|
||||||
|
let cargo_clippy_path = fs::canonicalize(format!("target/debug/cargo-clippy{EXE_SUFFIX}")).unwrap();
|
||||||
|
let clippy_driver_path = fs::canonicalize(format!("target/debug/clippy-driver{EXE_SUFFIX}")).unwrap();
|
||||||
|
|
||||||
// if the clippy bin is newer than our logs, throw away target dirs to force clippy to
|
// if the clippy bin is newer than our logs, throw away target dirs to force clippy to
|
||||||
// refresh the logs
|
// refresh the logs
|
||||||
if lintcheck_needs_rerun(&config.lintcheck_results_path) {
|
if lintcheck_needs_rerun(
|
||||||
|
&config.lintcheck_results_path,
|
||||||
|
[&cargo_clippy_path, &clippy_driver_path],
|
||||||
|
) {
|
||||||
let shared_target_dir = "target/lintcheck/shared_target_dir";
|
let shared_target_dir = "target/lintcheck/shared_target_dir";
|
||||||
// if we get an Err here, the shared target dir probably does simply not exist
|
// if we get an Err here, the shared target dir probably does simply not exist
|
||||||
if let Ok(metadata) = std::fs::metadata(&shared_target_dir) {
|
if let Ok(metadata) = std::fs::metadata(&shared_target_dir) {
|
||||||
|
@ -569,10 +611,6 @@ fn main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let cargo_clippy_path: PathBuf = PathBuf::from(CARGO_CLIPPY_PATH)
|
|
||||||
.canonicalize()
|
|
||||||
.expect("failed to canonicalize path to clippy binary");
|
|
||||||
|
|
||||||
// assert that clippy is found
|
// assert that clippy is found
|
||||||
assert!(
|
assert!(
|
||||||
cargo_clippy_path.is_file(),
|
cargo_clippy_path.is_file(),
|
||||||
|
@ -580,7 +618,7 @@ fn main() {
|
||||||
cargo_clippy_path.display()
|
cargo_clippy_path.display()
|
||||||
);
|
);
|
||||||
|
|
||||||
let clippy_ver = std::process::Command::new(CARGO_CLIPPY_PATH)
|
let clippy_ver = std::process::Command::new(&cargo_clippy_path)
|
||||||
.arg("--version")
|
.arg("--version")
|
||||||
.output()
|
.output()
|
||||||
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
|
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
|
||||||
|
@ -589,7 +627,7 @@ fn main() {
|
||||||
// download and extract the crates, then run clippy on them and collect clippy's warnings
|
// download and extract the crates, then run clippy on them and collect clippy's warnings
|
||||||
// flatten into one big list of warnings
|
// flatten into one big list of warnings
|
||||||
|
|
||||||
let crates = read_crates(&config.sources_toml_path);
|
let (crates, recursive_options) = read_crates(&config.sources_toml_path);
|
||||||
let old_stats = read_stats_from_file(&config.lintcheck_results_path);
|
let old_stats = read_stats_from_file(&config.lintcheck_results_path);
|
||||||
|
|
||||||
let counter = AtomicUsize::new(1);
|
let counter = AtomicUsize::new(1);
|
||||||
|
@ -639,11 +677,31 @@ fn main() {
|
||||||
.build_global()
|
.build_global()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let clippy_warnings: Vec<ClippyWarning> = crates
|
let server = config.recursive.then(|| {
|
||||||
|
let _ = fs::remove_dir_all("target/lintcheck/shared_target_dir/recursive");
|
||||||
|
|
||||||
|
LintcheckServer::spawn(recursive_options)
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut clippy_warnings: Vec<ClippyWarning> = crates
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.flat_map(|krate| krate.run_clippy_lints(&cargo_clippy_path, &counter, crates.len(), &config, &lint_filter))
|
.flat_map(|krate| {
|
||||||
|
krate.run_clippy_lints(
|
||||||
|
&cargo_clippy_path,
|
||||||
|
&clippy_driver_path,
|
||||||
|
&counter,
|
||||||
|
crates.len(),
|
||||||
|
&config,
|
||||||
|
&lint_filter,
|
||||||
|
&server,
|
||||||
|
)
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
if let Some(server) = server {
|
||||||
|
clippy_warnings.extend(server.warnings());
|
||||||
|
}
|
||||||
|
|
||||||
// if we are in --fix mode, don't change the log files, terminate here
|
// if we are in --fix mode, don't change the log files, terminate here
|
||||||
if config.fix {
|
if config.fix {
|
||||||
return;
|
return;
|
||||||
|
@ -681,8 +739,8 @@ fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
println!("Writing logs to {}", config.lintcheck_results_path.display());
|
println!("Writing logs to {}", config.lintcheck_results_path.display());
|
||||||
std::fs::create_dir_all(config.lintcheck_results_path.parent().unwrap()).unwrap();
|
fs::create_dir_all(config.lintcheck_results_path.parent().unwrap()).unwrap();
|
||||||
write(&config.lintcheck_results_path, text).unwrap();
|
fs::write(&config.lintcheck_results_path, text).unwrap();
|
||||||
|
|
||||||
print_stats(old_stats, new_stats, &config.lint_filter);
|
print_stats(old_stats, new_stats, &config.lint_filter);
|
||||||
}
|
}
|
||||||
|
|
123
lintcheck/src/recursive.rs
Normal file
123
lintcheck/src/recursive.rs
Normal file
|
@ -0,0 +1,123 @@
|
||||||
|
//! In `--recursive` mode we set the `lintcheck` binary as the `RUSTC_WRAPPER` of `cargo check`,
|
||||||
|
//! this allows [crate::driver] to be run for every dependency. The driver connects to
|
||||||
|
//! [LintcheckServer] to ask if it should be skipped, and if not sends the stderr of running clippy
|
||||||
|
//! on the crate to the server
|
||||||
|
|
||||||
|
use crate::ClippyWarning;
|
||||||
|
use crate::RecursiveOptions;
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::io::{BufRead, BufReader, Read, Write};
|
||||||
|
use std::net::{SocketAddr, TcpListener, TcpStream};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
use cargo_metadata::diagnostic::Diagnostic;
|
||||||
|
use crossbeam_channel::{Receiver, Sender};
|
||||||
|
use serde::de::DeserializeOwned;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Eq, Hash, PartialEq, Clone, Serialize, Deserialize)]
|
||||||
|
pub(crate) struct DriverInfo {
|
||||||
|
pub package_name: String,
|
||||||
|
pub crate_name: String,
|
||||||
|
pub version: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn serialize_line<T, W>(value: &T, writer: &mut W)
|
||||||
|
where
|
||||||
|
T: Serialize,
|
||||||
|
W: Write,
|
||||||
|
{
|
||||||
|
let mut buf = serde_json::to_vec(&value).expect("failed to serialize");
|
||||||
|
buf.push(b'\n');
|
||||||
|
writer.write_all(&buf).expect("write_all failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn deserialize_line<T, R>(reader: &mut R) -> T
|
||||||
|
where
|
||||||
|
T: DeserializeOwned,
|
||||||
|
R: BufRead,
|
||||||
|
{
|
||||||
|
let mut string = String::new();
|
||||||
|
reader.read_line(&mut string).expect("read_line failed");
|
||||||
|
serde_json::from_str(&string).expect("failed to deserialize")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_stream(
|
||||||
|
stream: TcpStream,
|
||||||
|
sender: &Sender<ClippyWarning>,
|
||||||
|
options: &RecursiveOptions,
|
||||||
|
seen: &Mutex<HashSet<DriverInfo>>,
|
||||||
|
) {
|
||||||
|
let mut stream = BufReader::new(stream);
|
||||||
|
|
||||||
|
let driver_info: DriverInfo = deserialize_line(&mut stream);
|
||||||
|
|
||||||
|
let unseen = seen.lock().unwrap().insert(driver_info.clone());
|
||||||
|
let ignored = options.ignore.contains(&driver_info.package_name);
|
||||||
|
let should_run = unseen && !ignored;
|
||||||
|
|
||||||
|
serialize_line(&should_run, stream.get_mut());
|
||||||
|
|
||||||
|
let mut stderr = String::new();
|
||||||
|
stream.read_to_string(&mut stderr).unwrap();
|
||||||
|
|
||||||
|
let messages = stderr
|
||||||
|
.lines()
|
||||||
|
.filter_map(|json_msg| serde_json::from_str::<Diagnostic>(json_msg).ok())
|
||||||
|
.filter_map(|diag| ClippyWarning::new(diag, &driver_info.package_name, &driver_info.version));
|
||||||
|
|
||||||
|
for message in messages {
|
||||||
|
sender.send(message).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct LintcheckServer {
|
||||||
|
pub local_addr: SocketAddr,
|
||||||
|
receiver: Receiver<ClippyWarning>,
|
||||||
|
sender: Arc<Sender<ClippyWarning>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LintcheckServer {
|
||||||
|
pub fn spawn(options: RecursiveOptions) -> Self {
|
||||||
|
let listener = TcpListener::bind("localhost:0").unwrap();
|
||||||
|
let local_addr = listener.local_addr().unwrap();
|
||||||
|
|
||||||
|
let (sender, receiver) = crossbeam_channel::unbounded::<ClippyWarning>();
|
||||||
|
let sender = Arc::new(sender);
|
||||||
|
// The spawned threads hold a `Weak<Sender>` so that they don't keep the channel connected
|
||||||
|
// indefinitely
|
||||||
|
let sender_weak = Arc::downgrade(&sender);
|
||||||
|
|
||||||
|
// Ignore dependencies multiple times, e.g. for when it's both checked and compiled for a
|
||||||
|
// build dependency
|
||||||
|
let seen = Mutex::default();
|
||||||
|
|
||||||
|
thread::spawn(move || {
|
||||||
|
thread::scope(|s| {
|
||||||
|
s.spawn(|| {
|
||||||
|
while let Ok((stream, _)) = listener.accept() {
|
||||||
|
let sender = sender_weak.upgrade().expect("received connection after server closed");
|
||||||
|
let options = &options;
|
||||||
|
let seen = &seen;
|
||||||
|
s.spawn(move || process_stream(stream, &sender, options, seen));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
Self {
|
||||||
|
local_addr,
|
||||||
|
sender,
|
||||||
|
receiver,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn warnings(self) -> impl Iterator<Item = ClippyWarning> {
|
||||||
|
// causes the channel to become disconnected so that the receiver iterator ends
|
||||||
|
drop(self.sender);
|
||||||
|
|
||||||
|
self.receiver.into_iter()
|
||||||
|
}
|
||||||
|
}
|
Loading…
Add table
Add a link
Reference in a new issue