Rollup merge of #133453 - ferrocene:check-license-metadata, r=Kobzol

Commit license-metadata.json to git and check it's correct in CI

This PR adds `license-metadata.json` to the root of the git repo, and changes `mingw-check` to check that the file is still up-to-date.

By committing this file, we remove the need for developers to a) have reuse installed or b) run an expensive ~90 second analysis of the files on disk when they want generate the COPYRIGHT.html files which depend on this license metadata.

The file will need updating whenever `REUSE.toml` changes, or when git submodules are added, or when git submodules change their license information (as detected by REUSE).

You can now run:

* `./x run collect-license-metadata` to update the `./license-metadata.json` file
* `./x test collect-license-metadata` to test the `./license-metadata.json` file for correctness

The comparison is done with two `serde_json::Value` objects, so the map objects they contain should ignore differences in ordering.

REUSE.toml

@@ -28,6 +28,7 @@ path = [
     "COPYRIGHT",
     "INSTALL.md",
     "LICENSE-APACHE",
+    "license-metadata.json",
     "LICENSE-MIT",
     "README.md",
     "RELEASES.md",

license-metadata.json

@@ -0,0 +1,269 @@
+{
+  "files": {
+    "children": [
+      {
+        "children": [
+          {
+            "children": [
+              {
+                "children": [
+                  {
+                    "children": [
+                      {
+                        "directories": [],
+                        "files": [
+                          "analyzer-decls.h",
+                          "malloc-macro.h"
+                        ],
+                        "license": {
+                          "copyright": [
+                            "2000-2024 Free Software Foundation, Inc"
+                          ],
+                          "spdx": "GPL-2.0-only"
+                        },
+                        "type": "group"
+                      }
+                    ],
+                    "license": {
+                      "copyright": [
+                        "2007-2011 Atheros Communications Inc",
+                        "2011-2012,2017 Qualcomm Atheros, Inc",
+                        "2016-2017 Erik Stromdahl <erik.stromdahl@gmail.com>"
+                      ],
+                      "spdx": "ISC"
+                    },
+                    "name": "c-c++-common/analyzer",
+                    "type": "directory"
+                  }
+                ],
+                "license": {
+                  "copyright": [
+                    "2000-2024 Free Software Foundation, Inc"
+                  ],
+                  "spdx": "GPL-2.0-only"
+                },
+                "name": "gcc/testsuite",
+                "type": "directory"
+              },
+              {
+                "license": {
+                  "copyright": [
+                    "2000-2024 Free Software Foundation, Inc"
+                  ],
+                  "spdx": "GCC-exception-3.1"
+                },
+                "name": "libstdc++-v3/config/os/aix/os_defines.h",
+                "type": "file"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "1997-2024 Free Software Foundation, Inc"
+              ],
+              "spdx": "GPL-3.0-or-later"
+            },
+            "name": "src/gcc",
+            "type": "directory"
+          },
+          {
+            "children": [
+              {
+                "license": {
+                  "copyright": [
+                    "The Rust Project Developers (see https://thanks.rust-lang.org)"
+                  ],
+                  "spdx": "Apache-2.0 OR MIT"
+                },
+                "name": "noscript.css",
+                "type": "file"
+              },
+              {
+                "license": {
+                  "copyright": [
+                    "Nicolas Gallagher and Jonathan Neal"
+                  ],
+                  "spdx": "MIT"
+                },
+                "name": "normalize.css",
+                "type": "file"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "2016 Ike Ku, Jessica Stokes and Leon Guan",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "src/librustdoc/html/static/css",
+            "type": "directory"
+          },
+          {
+            "children": [
+              {
+                "license": {
+                  "copyright": [
+                    "The Rust Project Developers (see https://thanks.rust-lang.org)"
+                  ],
+                  "spdx": "Apache-2.0 OR MIT"
+                },
+                "name": "README.txt",
+                "type": "file"
+              },
+              {
+                "directories": [],
+                "files": [
+                  "FiraSans-LICENSE.txt",
+                  "FiraSans-Medium.woff2",
+                  "FiraSans-Regular.woff2"
+                ],
+                "license": {
+                  "copyright": [
+                    "2014, Mozilla Foundation",
+                    "2014, Telefonica S.A"
+                  ],
+                  "spdx": "OFL-1.1"
+                },
+                "type": "group"
+              },
+              {
+                "directories": [],
+                "files": [
+                  "NanumBarunGothic-LICENSE.txt",
+                  "NanumBarunGothic.ttf.woff2"
+                ],
+                "license": {
+                  "copyright": [
+                    "2010 NAVER Corporation"
+                  ],
+                  "spdx": "OFL-1.1"
+                },
+                "type": "group"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "2010, 2012, 2014-2023, Adobe Systems Incorporated"
+              ],
+              "spdx": "OFL-1.1"
+            },
+            "name": "src/librustdoc/html/static/fonts",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2003-2019 University of Illinois at Urbana-Champaign",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 WITH LLVM-exception AND (Apache-2.0 OR MIT)"
+            },
+            "name": "compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2014 Alex Crichton",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "library/backtrace",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "1991-2024 Unicode, Inc"
+              ],
+              "spdx": "Unicode-3.0"
+            },
+            "name": "library/core/src/unicode/unicode_data.rs",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2019 The Crossbeam Project Developers",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "library/std/src/sync/mpmc",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2016 The Fuchsia Authors",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "BSD-2-Clause AND (Apache-2.0 OR MIT)"
+            },
+            "name": "library/std/src/sys/sync/mutex/fuchsia.rs",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "Rust on Embedded Devices Working Group",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR CC-BY-SA-4.0 OR MIT"
+            },
+            "name": "src/doc/embedded-book",
+            "type": "directory"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2014 Jorge Aparicio",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "src/doc/rust-by-example",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2014-2021 Knut Sveidqvist"
+              ],
+              "spdx": "MIT"
+            },
+            "name": "src/doc/rustc-dev-guide/mermaid.min.js",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2003-2019 University of Illinois at Urbana-Champaign",
+                "2003-2019 by the contributors listed in CREDITS.TXT (https://github.com/rust-lang/llvm-project/blob/7738295178045041669876bf32b0543ec8319a5c/llvm/CREDITS.TXT)",
+                "2010 Apple Inc"
+              ],
+              "spdx": "Apache-2.0 WITH LLVM-exception AND NCSA"
+            },
+            "name": "src/llvm-project",
+            "type": "directory"
+          }
+        ],
+        "license": {
+          "copyright": [
+            "The Rust Project Developers (see https://thanks.rust-lang.org)"
+          ],
+          "spdx": "Apache-2.0 OR MIT"
+        },
+        "name": ".",
+        "type": "directory"
+      }
+    ],
+    "type": "root"
+  }
+}

src/bootstrap/src/core/build_steps/run.rs

@@ -181,8 +181,7 @@ impl Step for CollectLicenseMetadata {
             panic!("REUSE is required to collect the license metadata");
         };
 
-        // Temporary location, it will be moved to src/etc once it's accurate.
+        let dest = builder.src.join("license-metadata.json");
-        let dest = builder.out.join("license-metadata.json");
 
         let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata);
         cmd.env("REUSE_EXE", reuse);
@@ -209,8 +208,7 @@ impl Step for GenerateCopyright {
     }
 
     fn run(self, builder: &Builder<'_>) -> Self::Output {
-        let license_metadata = builder.ensure(CollectLicenseMetadata);
+        let license_metadata = builder.src.join("license-metadata.json");
-
         let dest = builder.out.join("COPYRIGHT.html");
         let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 

src/bootstrap/src/core/build_steps/test.rs

@@ -3670,3 +3670,35 @@ impl Step for TestFloatParse {
         cargo_run.into_cmd().run(builder);
     }
 }
+
+#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+pub struct CollectLicenseMetadata;
+
+impl Step for CollectLicenseMetadata {
+    type Output = PathBuf;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/collect-license-metadata")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(CollectLicenseMetadata);
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let Some(reuse) = &builder.config.reuse else {
+            panic!("REUSE is required to collect the license metadata");
+        };
+
+        let dest = builder.src.join("license-metadata.json");
+
+        let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata);
+        cmd.env("REUSE_EXE", reuse);
+        cmd.env("DEST", &dest);
+        cmd.env("ONLY_CHECK", "1");
+        cmd.run(builder);
+
+        dest
+    }
+}

src/bootstrap/src/core/builder/mod.rs

@@ -915,6 +915,7 @@ impl<'a> Builder<'a> {
                 test::HtmlCheck,
                 test::RustInstaller,
                 test::TestFloatParse,
+                test::CollectLicenseMetadata,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -63,6 +63,7 @@ ENV SCRIPT \
            /scripts/validate-toolstate.sh && \
            /scripts/validate-error-codes.sh && \
            reuse --include-submodules lint && \
+           python3 ../x.py test collect-license-metadata && \
            # Runs checks to ensure that there are no issues in our JS code.
            es-check es2019 ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \

src/tools/collect-license-metadata/src/main.rs

@@ -4,7 +4,7 @@ mod reuse;
 
 use std::path::PathBuf;
 
-use anyhow::Error;
+use anyhow::{Context, Error};
 
 use crate::licenses::LicensesInterner;
 
@@ -12,10 +12,12 @@ use crate::licenses::LicensesInterner;
 ///
 /// You should probably let `bootstrap` execute this program instead of running it directly.
 ///
-/// Run `x.py run collect-license-metadata`
+/// * Run `x.py run collect-license-metadata` to re-regenerate the file.
+/// * Run `x.py test collect-license-metadata` to check if the file you have is correct.
 fn main() -> Result<(), Error> {
     let reuse_exe: PathBuf = std::env::var_os("REUSE_EXE").expect("Missing REUSE_EXE").into();
     let dest: PathBuf = std::env::var_os("DEST").expect("Missing DEST").into();
+    let only_check = std::env::var_os("ONLY_CHECK").is_some();
 
     let mut interner = LicensesInterner::new();
     let paths = crate::reuse::collect(&reuse_exe, &mut interner)?;
@@ -23,15 +25,32 @@ fn main() -> Result<(), Error> {
     let mut tree = crate::path_tree::build(paths);
     tree.simplify();
 
-    if let Some(parent) = dest.parent() {
+    let output = serde_json::json!({
-        std::fs::create_dir_all(parent)?;
+        "files": crate::path_tree::expand_interned_licenses(tree, &interner)
+    });
+
+    if only_check {
+        println!("loading existing license information");
+        let existing = std::fs::read_to_string(&dest).with_context(|| {
+            format!("Failed to read existing license JSON at {}", dest.display())
+        })?;
+        let existing_json: serde_json::Value =
+            serde_json::from_str(&existing).with_context(|| {
+                format!("Failed to read existing license JSON at {}", dest.display())
+            })?;
+        if existing_json != output {
+            eprintln!("The existing {} file is out of date.", dest.display());
+            eprintln!("Run ./x run collect-license-metadata to update it.");
+            anyhow::bail!("The existing {} file doesn't match what REUSE reports.", dest.display());
+        }
+        println!("license information matches");
+    } else {
+        if let Some(parent) = dest.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(&dest, &serde_json::to_vec_pretty(&output)?)?;
+        println!("license information written to {}", dest.display());
     }
-    std::fs::write(
-        &dest,
-        &serde_json::to_vec_pretty(&serde_json::json!({
-            "files": crate::path_tree::expand_interned_licenses(tree, &interner),
-        }))?,
-    )?;
 
     Ok(())
 }

src/tools/collect-license-metadata/src/reuse.rs

@@ -10,10 +10,10 @@ pub(crate) fn collect(
     reuse_exe: &Path,
     interner: &mut LicensesInterner,
 ) -> Result<Vec<(PathBuf, LicenseId)>, Error> {
-    eprintln!("gathering license information from REUSE");
+    println!("gathering license information from REUSE (this might take a minute...)");
     let start = Instant::now();
     let raw = &obtain_spdx_document(reuse_exe)?;
-    eprintln!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
+    println!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
 
     let document = spdx_rs::parsers::spdx_from_tag_value(&raw)?;
 

src/tools/generate-copyright/src/main.rs

@@ -57,6 +57,10 @@ fn main() -> Result<(), Error> {
         dependencies: collected_cargo_metadata,
     };
     let output = template.render()?;
+    // Git stores text files with \n, but this file may contain \r\n in files
+    // copied from dependencies. Normalise them before we write them out, for
+    // consistency.
+    let output = output.replace("\r\n", "\n");
     std::fs::write(&dest_file, output)?;
 
     // Output libstd subset file
@@ -65,6 +69,10 @@ fn main() -> Result<(), Error> {
         dependencies: library_collected_cargo_metadata,
     };
     let output = template.render()?;
+    // Git stores text files with \n, but this file may contain \r\n in files
+    // copied from dependencies. Normalise them before we write them out, for
+    // consistency.
+    let output = output.replace("\r\n", "\n");
     std::fs::write(&libstd_dest_file, output)?;
 
     Ok(())