bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rollup merge of #69443 - ehuss:tidy-license, r=skade,Mark-Simulacrum

Browse source 
tidy: Better license checks.

This implements some improvements to the license checks in tidy:

* Use `cargo_metadata` instead of parsing vendored crates. This allows license checks to run without vendoring enabled, and allows the checks to run on PR builds.
* Check for stale entries.
* Check that the licenses for exceptions are what we think they are.
* Verify exceptions do not leak into the runtime.

Closes #62618
Closes #62619
Closes #63238 (I think)

There are some substantive changes here. The follow licenses have changed from the original comments:

* openssl BSD+advertising clause to Apache-2.0
* pest MPL2 to MIT/Apache-2.0
* smallvec MPL2 to MIT/Apache-2.0
* clippy lints MPL2 to MIT OR Apache-2.0
This commit is contained in:
Mazdak Farrokhzad 2020-03-19 06:57:30 +01:00 • committed by GitHub
commit 61fe2e4036
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 365 additions and 353 deletions
Download patch file Download diff file Expand all files Collapse all files

13
Cargo.lock
View file

@ -386,9 +386,9 @@ dependencies = [
[[package]] [[package]]
name = "cargo_metadata" name = "cargo_metadata"
version = "0.9.0" version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d2d1617e838936c0d2323a65cc151e03ae19a7678dd24f72bccf27119b90a5d" checksum = "46e3374c604fb39d1a2f35ed5e4a4e30e60d01fab49446e08f1b3e9a90aef202"
dependencies = [ dependencies = [
"semver", "semver",
"serde", "serde",
@ -450,7 +450,7 @@ dependencies = [
name = "clippy" name = "clippy"
version = "0.0.212" version = "0.0.212"
dependencies = [ dependencies = [
"cargo_metadata 0.9.0", "cargo_metadata 0.9.1",
"clippy-mini-macro-test", "clippy-mini-macro-test",
"clippy_lints", "clippy_lints",
"compiletest_rs", "compiletest_rs",
@ -473,7 +473,7 @@ version = "0.2.0"
name = "clippy_lints" name = "clippy_lints"
version = "0.0.212" version = "0.0.212"
dependencies = [ dependencies = [
"cargo_metadata 0.9.0", "cargo_metadata 0.9.1",
"if_chain", "if_chain",
"itertools 0.9.0", "itertools 0.9.0",
"lazy_static 1.4.0", "lazy_static 1.4.0",
@ -2132,7 +2132,7 @@ name = "miri"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"cargo_metadata 0.9.0", "cargo_metadata 0.9.1",
"colored", "colored",
"compiletest_rs", "compiletest_rs",
"directories", "directories",
@ -4860,10 +4860,9 @@ dependencies = [
name = "tidy" name = "tidy"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"cargo_metadata 0.9.1",
"lazy_static 1.4.0", "lazy_static 1.4.0",
"regex", "regex",
"serde",
"serde_json",
"walkdir", "walkdir",
] ]

3
src/bootstrap/test.rs
View file

@ -728,9 +728,6 @@ impl Step for Tidy {
let mut cmd = builder.tool_cmd(Tool::Tidy); let mut cmd = builder.tool_cmd(Tool::Tidy);
cmd.arg(builder.src.join("src")); cmd.arg(builder.src.join("src"));
cmd.arg(&builder.initial_cargo); cmd.arg(&builder.initial_cargo);
if !builder.config.vendor {
cmd.arg("--no-vendor");
}
if builder.is_verbose() { if builder.is_verbose() {
cmd.arg("--verbose"); cmd.arg("--verbose");
} }

3
src/tools/tidy/Cargo.toml
View file

@ -5,8 +5,7 @@ authors = ["Alex Crichton <alex@alexcrichton.com>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
cargo_metadata = "0.9.1"
regex = "1" regex = "1"
serde = { version = "1.0.8", features = ["derive"] }
serde_json = "1.0.2"
lazy_static = "1" lazy_static = "1"
walkdir = "2" walkdir = "2"

694
src/tools/tidy/src/deps.rs
View file

@ -1,13 +1,11 @@
//! Checks the licenses of third-party dependencies by inspecting vendors. //! Checks the licenses of third-party dependencies.
use std::collections::{BTreeSet, HashMap, HashSet}; use cargo_metadata::{Metadata, Package, PackageId, Resolve};
use std::fs; use std::collections::{BTreeSet, HashSet};
use std::path::Path; use std::path::Path;
use std::process::Command;
use serde::Deserialize;
use serde_json;
/// These are licenses that are allowed for all crates, including the runtime,
/// rustc, tools, etc.
const LICENSES: &[&str] = &[ const LICENSES: &[&str] = &[
"MIT/Apache-2.0", "MIT/Apache-2.0",
"MIT / Apache-2.0", "MIT / Apache-2.0",
@ -25,261 +23,268 @@ const LICENSES: &[&str] = &[
/// should be considered bugs. Exceptions are only allowed in Rust /// should be considered bugs. Exceptions are only allowed in Rust
/// tooling. It is _crucial_ that no exception crates be dependencies /// tooling. It is _crucial_ that no exception crates be dependencies
/// of the Rust runtime (std/test). /// of the Rust runtime (std/test).
const EXCEPTIONS: &[&str] = &[ const EXCEPTIONS: &[(&str, &str)] = &[
"mdbook", // MPL2, mdbook ("mdbook", "MPL-2.0"), // mdbook
"openssl", // BSD+advertising clause, cargo, mdbook ("openssl", "Apache-2.0"), // cargo, mdbook
"pest", // MPL2, mdbook via handlebars ("arrayref", "BSD-2-Clause"), // mdbook via handlebars via pest
"arrayref", // BSD-2-Clause, mdbook via handlebars via pest ("toml-query", "MPL-2.0"), // mdbook
"thread-id", // Apache-2.0, mdbook ("toml-query_derive", "MPL-2.0"), // mdbook
"toml-query", // MPL-2.0, mdbook ("is-match", "MPL-2.0"), // mdbook
"is-match", // MPL-2.0, mdbook ("rdrand", "ISC"), // mdbook, rustfmt
"cssparser", // MPL-2.0, rustdoc ("fuchsia-cprng", "BSD-3-Clause"), // mdbook, rustfmt
"smallvec", // MPL-2.0, rustdoc ("fuchsia-zircon-sys", "BSD-3-Clause"), // rustdoc, rustc, cargo
"rdrand", // ISC, mdbook, rustfmt ("fuchsia-zircon", "BSD-3-Clause"), // rustdoc, rustc, cargo (jobserver & tempdir)
"fuchsia-cprng", // BSD-3-Clause, mdbook, rustfmt ("colored", "MPL-2.0"), // rustfmt
"fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo ("ordslice", "Apache-2.0"), // rls
"fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir) ("cloudabi", "BSD-2-Clause"), // (rls -> crossbeam-channel 0.2 -> rand 0.5)
"cssparser-macros", // MPL-2.0, rustdoc ("ryu", "Apache-2.0 OR BSL-1.0"), // rls/cargo/... (because of serde)
"selectors", // MPL-2.0, rustdoc ("bytesize", "Apache-2.0"), // cargo
"clippy_lints", // MPL-2.0, rls ("im-rc", "MPL-2.0+"), // cargo
"colored", // MPL-2.0, rustfmt ("adler32", "BSD-3-Clause AND Zlib"), // cargo dep that isn't used
"ordslice", // Apache-2.0, rls ("constant_time_eq", "CC0-1.0"), // rustfmt
"cloudabi", // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5) ("sized-chunks", "MPL-2.0+"), // cargo via im-rc
"ryu", // Apache-2.0, rls/cargo/... (because of serde) ("bitmaps", "MPL-2.0+"), // cargo via im-rc
"bytesize", // Apache-2.0, cargo
"im-rc", // MPL-2.0+, cargo
"adler32", // BSD-3-Clause AND Zlib, cargo dep that isn't used
"constant_time_eq", // CC0-1.0, rustfmt
"utf8parse", // Apache-2.0 OR MIT, cargo via strip-ansi-escapes
"vte", // Apache-2.0 OR MIT, cargo via strip-ansi-escapes
"sized-chunks", // MPL-2.0+, cargo via im-rc
"bitmaps", // MPL-2.0+, cargo via im-rc
// FIXME: this dependency violates the documentation comment above: // FIXME: this dependency violates the documentation comment above:
"fortanix-sgx-abi", // MPL-2.0+, libstd but only for `sgx` target ("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target
"dunce", // CC0-1.0 mdbook-linkcheck ("dunce", "CC0-1.0"), // mdbook-linkcheck
"codespan-reporting", // Apache-2.0 mdbook-linkcheck ("codespan-reporting", "Apache-2.0"), // mdbook-linkcheck
"codespan", // Apache-2.0 mdbook-linkcheck ("codespan", "Apache-2.0"), // mdbook-linkcheck
"crossbeam-channel", // MIT/Apache-2.0 AND BSD-2-Clause, cargo ("crossbeam-channel", "MIT/Apache-2.0 AND BSD-2-Clause"), // cargo
]; ];
/// These are the root crates that are part of the runtime. The licenses for
/// these and all their dependencies *must not* be in the exception list.
const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", "panic_unwind"];
/// Which crates to check against the whitelist? /// Which crates to check against the whitelist?
const WHITELIST_CRATES: &[CrateVersion<'_>] = const WHITELIST_CRATES: &[&str] = &["rustc", "rustc_codegen_llvm"];
&[CrateVersion("rustc", "0.0.0"), CrateVersion("rustc_codegen_llvm", "0.0.0")];
/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
const WHITELIST: &[Crate<'_>] = &[ ///
Crate("adler32"), /// This list is here to provide a speed-bump to adding a new dependency to
Crate("aho-corasick"), /// rustc. Please check with the compiler team before adding an entry.
Crate("annotate-snippets"), const WHITELIST: &[&str] = &[
Crate("ansi_term"), "adler32",
Crate("arrayvec"), "aho-corasick",
Crate("atty"), "annotate-snippets",
Crate("autocfg"), "ansi_term",
Crate("backtrace"), "arrayvec",
Crate("backtrace-sys"), "atty",
Crate("bitflags"), "autocfg",
Crate("build_const"), "backtrace",
Crate("byteorder"), "backtrace-sys",
Crate("c2-chacha"), "bitflags",
Crate("cc"), "byteorder",
Crate("cfg-if"), "c2-chacha",
Crate("chalk-engine"), "cc",
Crate("chalk-macros"), "cfg-if",
Crate("cloudabi"), "cloudabi",
Crate("cmake"), "cmake",
Crate("compiler_builtins"), "compiler_builtins",
Crate("crc"), "crc32fast",
Crate("crc32fast"), "crossbeam-deque",
Crate("crossbeam-deque"), "crossbeam-epoch",
Crate("crossbeam-epoch"), "crossbeam-queue",
Crate("crossbeam-queue"), "crossbeam-utils",
Crate("crossbeam-utils"), "datafrog",
Crate("datafrog"), "dlmalloc",
Crate("dlmalloc"), "either",
Crate("either"), "ena",
Crate("ena"), "env_logger",
Crate("env_logger"), "filetime",
Crate("filetime"), "flate2",
Crate("flate2"), "fortanix-sgx-abi",
Crate("fortanix-sgx-abi"), "fuchsia-zircon",
Crate("fuchsia-zircon"), "fuchsia-zircon-sys",
Crate("fuchsia-zircon-sys"), "getopts",
Crate("getopts"), "getrandom",
Crate("getrandom"), "hashbrown",
Crate("hashbrown"), "hermit-abi",
Crate("humantime"), "humantime",
Crate("indexmap"), "indexmap",
Crate("itertools"), "itertools",
Crate("jobserver"), "jobserver",
Crate("kernel32-sys"), "kernel32-sys",
Crate("lazy_static"), "lazy_static",
Crate("libc"), "libc",
Crate("libz-sys"), "libz-sys",
Crate("lock_api"), "lock_api",
Crate("log"), "log",
Crate("log_settings"), "log_settings",
Crate("measureme"), "measureme",
Crate("memchr"), "memchr",
Crate("memmap"), "memmap",
Crate("memoffset"), "memoffset",
Crate("miniz-sys"), "miniz_oxide",
Crate("miniz_oxide"), "nodrop",
Crate("miniz_oxide_c_api"), "num_cpus",
Crate("nodrop"), "parking_lot",
Crate("num_cpus"), "parking_lot_core",
Crate("owning_ref"), "pkg-config",
Crate("parking_lot"), "polonius-engine",
Crate("parking_lot_core"), "ppv-lite86",
Crate("pkg-config"), "proc-macro2",
Crate("polonius-engine"), "punycode",
Crate("ppv-lite86"), "quick-error",
Crate("proc-macro2"), "quote",
Crate("punycode"), "rand",
Crate("quick-error"), "rand_chacha",
Crate("quote"), "rand_core",
Crate("rand"), "rand_hc",
Crate("rand_chacha"), "rand_isaac",
Crate("rand_core"), "rand_pcg",
Crate("rand_hc"), "rand_xorshift",
Crate("rand_isaac"), "redox_syscall",
Crate("rand_pcg"), "redox_termios",
Crate("rand_xorshift"), "regex",
Crate("redox_syscall"), "regex-syntax",
Crate("redox_termios"), "remove_dir_all",
Crate("regex"), "rustc-demangle",
Crate("regex-syntax"), "rustc-hash",
Crate("remove_dir_all"), "rustc-rayon",
Crate("rustc-demangle"), "rustc-rayon-core",
Crate("rustc-hash"), "rustc_version",
Crate("rustc-rayon"), "scoped-tls",
Crate("rustc-rayon-core"), "scopeguard",
Crate("rustc_version"), "semver",
Crate("scoped-tls"), "semver-parser",
Crate("scopeguard"), "serde",
Crate("semver"), "serde_derive",
Crate("semver-parser"), "smallvec",
Crate("serde"), "stable_deref_trait",
Crate("serde_derive"), "syn",
Crate("smallvec"), "synstructure",
Crate("stable_deref_trait"), "tempfile",
Crate("syn"), "termcolor",
Crate("synstructure"), "termion",
Crate("tempfile"), "termize",
Crate("termcolor"), "thread_local",
Crate("terminon"), "ucd-util",
Crate("termion"), "unicode-normalization",
Crate("termize"), "unicode-script",
Crate("thread_local"), "unicode-security",
Crate("ucd-util"), "unicode-width",
Crate("unicode-normalization"), "unicode-xid",
Crate("unicode-script"), "utf8-ranges",
Crate("unicode-security"), "vcpkg",
Crate("unicode-width"), "version_check",
Crate("unicode-xid"), "wasi",
Crate("unreachable"), "winapi",
Crate("utf8-ranges"), "winapi-build",
Crate("vcpkg"), "winapi-i686-pc-windows-gnu",
Crate("version_check"), "winapi-util",
Crate("void"), "winapi-x86_64-pc-windows-gnu",
Crate("wasi"), "wincolor",
Crate("winapi"),
Crate("winapi-build"),
Crate("winapi-i686-pc-windows-gnu"),
Crate("winapi-util"),
Crate("winapi-x86_64-pc-windows-gnu"),
Crate("wincolor"),
Crate("hermit-abi"),
]; ];
// Some types for Serde to deserialize the output of `cargo metadata` to. /// Dependency checks.
#[derive(Deserialize)]
struct Output {
resolve: Resolve,
}
#[derive(Deserialize)]
struct Resolve {
nodes: Vec<ResolveNode>,
}
#[derive(Deserialize)]
struct ResolveNode {
id: String,
dependencies: Vec<String>,
}
/// A unique identifier for a crate.
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct Crate<'a>(&'a str); // (name)
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct CrateVersion<'a>(&'a str, &'a str); // (name, version)
impl Crate<'_> {
pub fn id_str(&self) -> String {
format!("{} ", self.0)
}
}
impl<'a> CrateVersion<'a> {
/// Returns the struct and whether or not the dependency is in-tree.
pub fn from_str(s: &'a str) -> (Self, bool) {
let mut parts = s.split(' ');
let name = parts.next().unwrap();
let version = parts.next().unwrap();
let path = parts.next().unwrap();
let is_path_dep = path.starts_with("(path+");
(CrateVersion(name, version), is_path_dep)
}
pub fn id_str(&self) -> String {
format!("{} {}", self.0, self.1)
}
}
impl<'a> From<CrateVersion<'a>> for Crate<'a> {
fn from(cv: CrateVersion<'a>) -> Crate<'a> {
Crate(cv.0)
}
}
/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
/// ///
/// Specifically, this checks that the license is correct. /// `path` is path to the `src` directory, `cargo` is path to the cargo executable.
pub fn check(path: &Path, bad: &mut bool) { pub fn check(path: &Path, cargo: &Path, bad: &mut bool) {
// Check licences. let mut cmd = cargo_metadata::MetadataCommand::new();
let path = path.join("../vendor"); cmd.cargo_path(cargo)
assert!(path.exists(), "vendor directory missing"); .manifest_path(path.parent().unwrap().join("Cargo.toml"))
let mut saw_dir = false; .features(cargo_metadata::CargoOpt::AllFeatures);
for dir in t!(path.read_dir()) { let metadata = t!(cmd.exec());
saw_dir = true; check_exceptions(&metadata, bad);
let dir = t!(dir); check_whitelist(&metadata, bad);
check_crate_duplicate(&metadata, bad);
}
// Skip our exceptions. /// Check that all licenses are in the valid list in `LICENSES`.
let is_exception = EXCEPTIONS.iter().any(|exception| { ///
dir.path().to_str().unwrap().contains(&format!("vendor/{}", exception)) /// Packages listed in `EXCEPTIONS` are allowed for tools.
}); fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
if is_exception { // Validate the EXCEPTIONS list hasn't changed.
for (name, license) in EXCEPTIONS {
// Check that the package actually exists.
if !metadata.packages.iter().any(|p| p.name == *name) {
println!(
"could not find exception package `{}`\n\
Remove from EXCEPTIONS list if it is no longer used.",
name
);
*bad = true;
}
// Check that the license hasn't changed.
for pkg in metadata.packages.iter().filter(|p| p.name == *name) {
if pkg.name == "fuchsia-cprng" {
// This package doesn't declare a license expression. Manual
// inspection of the license file is necessary, which appears
// to be BSD-3-Clause.
assert!(pkg.license.is_none());
continue;
}
match &pkg.license {
None => {
println!(
"dependency exception `{}` does not declare a license expression",
pkg.id
);
*bad = true;
}
Some(pkg_license) => {
if pkg_license.as_str() != *license {
println!("dependency exception `{}` license has changed", name);
println!(" previously `{}` now `{}`", license, pkg_license);
println!(" update EXCEPTIONS for the new license");
*bad = true;
}
}
}
}
}
let exception_names: Vec<_> = EXCEPTIONS.iter().map(|(name, _license)| *name).collect();
let runtime_ids = compute_runtime_crates(metadata);
// Check if any package does not have a valid license.
for pkg in &metadata.packages {
if pkg.source.is_none() {
// No need to check local packages.
continue; continue;
} }
if !runtime_ids.contains(&pkg.id) && exception_names.contains(&pkg.name.as_str()) {
let toml = dir.path().join("Cargo.toml"); continue;
*bad = !check_license(&toml) || *bad; }
let license = match &pkg.license {
Some(license) => license,
None => {
println!("dependency `{}` does not define a license expression", pkg.id,);
*bad = true;
continue;
}
};
if !LICENSES.contains(&license.as_str()) {
if pkg.name == "fortanix-sgx-abi" {
// This is a specific exception because SGX is considered
// "third party". See
// https://github.com/rust-lang/rust/issues/62620 for more. In
// general, these should never be added.
continue;
}
println!("invalid license `{}` in `{}`", license, pkg.id);
*bad = true;
}
} }
assert!(saw_dir, "no vendored source");
} }
/// Checks the dependency of `WHITELIST_CRATES` at the given path. Changes `bad` to `true` if a /// Checks the dependency of `WHITELIST_CRATES` at the given path. Changes `bad` to `true` if a
/// check failed. /// check failed.
/// ///
/// Specifically, this checks that the dependencies are on the `WHITELIST`. /// Specifically, this checks that the dependencies are on the `WHITELIST`.
pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { fn check_whitelist(metadata: &Metadata, bad: &mut bool) {
// Get dependencies from Cargo metadata. // Check that the WHITELIST does not have unused entries.
let resolve = get_deps(path, cargo); for name in WHITELIST {
if !metadata.packages.iter().any(|p| p.name == *name) {
println!(
"could not find whitelisted package `{}`\n\
Remove from WHITELIST list if it is no longer used.",
name
);
*bad = true;
}
}
// Get the whitelist in a convenient form. // Get the whitelist in a convenient form.
let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect(); let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();
@ -287,122 +292,59 @@ pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
let mut visited = BTreeSet::new(); let mut visited = BTreeSet::new();
let mut unapproved = BTreeSet::new(); let mut unapproved = BTreeSet::new();
for &krate in WHITELIST_CRATES.iter() { for &krate in WHITELIST_CRATES.iter() {
let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false); let pkg = pkg_from_name(metadata, krate);
let mut bad = check_crate_whitelist(&whitelist, metadata, &mut visited, pkg);
unapproved.append(&mut bad); unapproved.append(&mut bad);
} }
if !unapproved.is_empty() { if !unapproved.is_empty() {
println!("Dependencies not on the whitelist:"); println!("Dependencies not on the whitelist:");
for dep in unapproved { for dep in unapproved {
println!("* {}", dep.id_str()); println!("* {}", dep);
} }
*bad = true; *bad = true;
} }
check_crate_duplicate(&resolve, bad);
}
fn check_license(path: &Path) -> bool {
if !path.exists() {
panic!("{} does not exist", path.display());
}
let contents = t!(fs::read_to_string(&path));
let mut found_license = false;
for line in contents.lines() {
if !line.starts_with("license") {
continue;
}
let license = extract_license(line);
if !LICENSES.contains(&&*license) {
println!("invalid license {} in {}", license, path.display());
return false;
}
found_license = true;
break;
}
if !found_license {
println!("no license in {}", path.display());
return false;
}
true
}
fn extract_license(line: &str) -> String {
let first_quote = line.find('"');
let last_quote = line.rfind('"');
if let (Some(f), Some(l)) = (first_quote, last_quote) {
let license = &line[f + 1..l];
license.into()
} else {
"bad-license-parse".into()
}
}
/// Gets the dependencies of the crate at the given path using `cargo metadata`.
fn get_deps(path: &Path, cargo: &Path) -> Resolve {
// Run `cargo metadata` to get the set of dependencies.
let output = Command::new(cargo)
.arg("metadata")
.arg("--format-version")
.arg("1")
.arg("--manifest-path")
.arg(path.join("../Cargo.toml"))
.output()
.expect("Unable to run `cargo metadata`")
.stdout;
let output = String::from_utf8_lossy(&output);
let output: Output = serde_json::from_str(&output).unwrap();
output.resolve
} }
/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on /// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
/// the whitelist. Returns a list of illegal dependencies. /// the whitelist. Returns a list of illegal dependencies.
fn check_crate_whitelist<'a>( fn check_crate_whitelist<'a>(
whitelist: &'a HashSet<Crate<'_>>, whitelist: &'a HashSet<&'static str>,
resolve: &'a Resolve, metadata: &'a Metadata,
visited: &mut BTreeSet<CrateVersion<'a>>, visited: &mut BTreeSet<&'a PackageId>,
krate: CrateVersion<'a>, krate: &'a Package,
must_be_on_whitelist: bool, ) -> BTreeSet<&'a PackageId> {
) -> BTreeSet<Crate<'a>> {
// This will contain bad deps. // This will contain bad deps.
let mut unapproved = BTreeSet::new(); let mut unapproved = BTreeSet::new();
// Check if we have already visited this crate. // Check if we have already visited this crate.
if visited.contains(&krate) { if visited.contains(&krate.id) {
return unapproved; return unapproved;
} }
visited.insert(krate); visited.insert(&krate.id);
// If this path is in-tree, we don't require it to be on the whitelist. // If this path is in-tree, we don't require it to be on the whitelist.
if must_be_on_whitelist { if krate.source.is_some() {
// If this dependency is not on `WHITELIST`, add to bad set. // If this dependency is not on `WHITELIST`, add to bad set.
if !whitelist.contains(&krate.into()) { if !whitelist.contains(krate.name.as_str()) {
unapproved.insert(krate.into()); unapproved.insert(&krate.id);
} }
} }
// Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!). // Do a DFS in the crate graph.
let to_check = resolve let to_check = deps_of(metadata, &krate.id);
.nodes
.iter()
.find(|n| n.id.starts_with(&krate.id_str()))
.expect("crate does not exist");
for dep in to_check.dependencies.iter() { for dep in to_check {
let (krate, is_path_dep) = CrateVersion::from_str(dep); let mut bad = check_crate_whitelist(whitelist, metadata, visited, dep);
let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
unapproved.append(&mut bad); unapproved.append(&mut bad);
} }
unapproved unapproved
} }
fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) { /// Prevents multiple versions of some expensive crates.
fn check_crate_duplicate(metadata: &Metadata, bad: &mut bool) {
const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[ const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
// These two crates take quite a long time to build, so don't allow two versions of them // These two crates take quite a long time to build, so don't allow two versions of them
// to accidentally sneak into our dependency graph, in order to ensure we keep our CI times // to accidentally sneak into our dependency graph, in order to ensure we keep our CI times
@ -410,19 +352,97 @@ fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
"cargo", "cargo",
"rustc-ap-syntax", "rustc-ap-syntax",
]; ];
let mut name_to_id: HashMap<_, Vec<_>> = HashMap::new();
for node in resolve.nodes.iter() {
name_to_id.entry(node.id.split_whitespace().next().unwrap()).or_default().push(&node.id);
}
for name in FORBIDDEN_TO_HAVE_DUPLICATES { for &name in FORBIDDEN_TO_HAVE_DUPLICATES {
if name_to_id[name].len() <= 1 { let matches: Vec<_> = metadata.packages.iter().filter(|pkg| pkg.name == name).collect();
continue; match matches.len() {
0 => {
println!(
"crate `{}` is missing, update `check_crate_duplicate` \
if it is no longer used",
name
);
*bad = true;
}
1 => {}
_ => {
println!(
"crate `{}` is duplicated in `Cargo.lock`, \
it is too expensive to build multiple times, \
so make sure only one version appears across all dependencies",
name
);
for pkg in matches {
println!(" * {}", pkg.id);
}
*bad = true;
}
} }
println!("crate `{}` is duplicated in `Cargo.lock`", name); }
for id in name_to_id[name].iter() { }
println!(" * {}", id);
} /// Returns a list of dependencies for the given package.
*bad = true; fn deps_of<'a>(metadata: &'a Metadata, pkg_id: &'a PackageId) -> Vec<&'a Package> {
let resolve = metadata.resolve.as_ref().unwrap();
let node = resolve
.nodes
.iter()
.find(|n| &n.id == pkg_id)
.unwrap_or_else(|| panic!("could not find `{}` in resolve", pkg_id));
node.deps
.iter()
.map(|dep| {
metadata.packages.iter().find(|pkg| pkg.id == dep.pkg).unwrap_or_else(|| {
panic!("could not find dep `{}` for pkg `{}` in resolve", dep.pkg, pkg_id)
})
})
.collect()
}
/// Finds a package with the given name.
fn pkg_from_name<'a>(metadata: &'a Metadata, name: &'static str) -> &'a Package {
let mut i = metadata.packages.iter().filter(|p| p.name == name);
let result =
i.next().unwrap_or_else(|| panic!("could not find package `{}` in package list", name));
assert!(i.next().is_none(), "more than one package found for `{}`", name);
result
}
/// Finds all the packages that are in the rust runtime.
fn compute_runtime_crates<'a>(metadata: &'a Metadata) -> HashSet<&'a PackageId> {
let resolve = metadata.resolve.as_ref().unwrap();
let mut result = HashSet::new();
for name in RUNTIME_CRATES {
let id = &pkg_from_name(metadata, name).id;
normal_deps_of_r(resolve, id, &mut result);
}
result
}
/// Recursively find all normal dependencies.
fn normal_deps_of_r<'a>(
resolve: &'a Resolve,
pkg_id: &'a PackageId,
result: &mut HashSet<&'a PackageId>,
) {
if !result.insert(pkg_id) {
return;
}
let node = resolve
.nodes
.iter()
.find(|n| &n.id == pkg_id)
.unwrap_or_else(|| panic!("could not find `{}` in resolve", pkg_id));
// Don't care about dev-dependencies.
// Build dependencies *shouldn't* matter unless they do some kind of
// codegen. For now we'll assume they don't.
let deps = node.deps.iter().filter(|node_dep| {
node_dep
.dep_kinds
.iter()
.any(|kind_info| kind_info.kind == cargo_metadata::DependencyKind::Normal)
});
for dep in deps {
normal_deps_of_r(resolve, &dep.pkg, result);
} }
} }

5
src/tools/tidy/src/main.rs
View file

@ -30,10 +30,7 @@ fn main() {
pal::check(&path, &mut bad); pal::check(&path, &mut bad);
unstable_book::check(&path, collected, &mut bad); unstable_book::check(&path, collected, &mut bad);
unit_tests::check(&path, &mut bad); unit_tests::check(&path, &mut bad);
if !args.iter().any(|s| *s == "--no-vendor") { deps::check(&path, &cargo, &mut bad);
deps::check(&path, &mut bad);
}
deps::check_whitelist(&path, &cargo, &mut bad);
extdeps::check(&path, &mut bad); extdeps::check(&path, &mut bad);
ui_tests::check(&path, &mut bad); ui_tests::check(&path, &mut bad);
error_codes_check::check(&path, &mut bad); error_codes_check::check(&path, &mut bad);