Move download-ci-llvm to rustbuild

This attempts to keep the logic as close to the original python as possible.
`probably_large` has been removed, since it was always `True`, and UTF-8 paths are no longer supported when patching files for NixOS.
I can readd UTF-8 support if desired.

Note that this required making `llvm_link_shared` computed on-demand,
since we don't know whether it will be static or dynamic until we download LLVM from CI.

Cargo.lock

@@ -225,8 +225,11 @@ dependencies = [
  "pretty_assertions",
  "serde",
  "serde_json",
+ "tar",
+ "tempfile",
  "toml",
  "winapi",
+ "xz2",
 ]
 
 [[package]]

src/bootstrap/Cargo.toml

@@ -42,10 +42,13 @@ cc = "1.0.69"
 libc = "0.2"
 serde = { version = "1.0.8", features = ["derive"] }
 serde_json = "1.0.2"
+tar = "0.4"
+tempfile = "3"
 toml = "0.5"
 ignore = "0.4.10"
 opener = "0.5"
 once_cell = "1.7.2"
+xz2 = "0.1"
 
 [target.'cfg(windows)'.dependencies.winapi]
 version = "0.3"

src/bootstrap/bootstrap.py

@@ -500,81 +500,6 @@ class RustBuild(object):
                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 
-        # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
-        if self.downloading_llvm() and stage0:
-            # We want the most recent LLVM submodule update to avoid downloading
-            # LLVM more often than necessary.
-            #
-            # This git command finds that commit SHA, looking for bors-authored
-            # commits that modified src/llvm-project or other relevant version
-            # stamp files.
-            #
-            # This works even in a repository that has not yet initialized
-            # submodules.
-            top_level = subprocess.check_output([
-                "git", "rev-parse", "--show-toplevel",
-            ]).decode(sys.getdefaultencoding()).strip()
-            llvm_sha = subprocess.check_output([
-                "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
-                "--first-parent", "HEAD",
-                "--",
-                "{}/src/llvm-project".format(top_level),
-                "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
-                # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-                "{}/src/version".format(top_level)
-            ]).decode(sys.getdefaultencoding()).strip()
-            llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
-            llvm_root = self.llvm_root()
-            llvm_lib = os.path.join(llvm_root, "lib")
-            if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
-                self._download_ci_llvm(llvm_sha, llvm_assertions)
-                for binary in ["llvm-config", "FileCheck"]:
-                    self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
-                for lib in os.listdir(llvm_lib):
-                    if lib.endswith(".so"):
-                        self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
-                with output(self.llvm_stamp()) as llvm_stamp:
-                    llvm_stamp.write(llvm_sha + str(llvm_assertions))
-
-    def downloading_llvm(self):
-        opt = self.get_toml('download-ci-llvm', 'llvm')
-        # This is currently all tier 1 targets and tier 2 targets with host tools
-        # (since others may not have CI artifacts)
-        # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
-        supported_platforms = [
-            # tier 1
-            "aarch64-unknown-linux-gnu",
-            "i686-pc-windows-gnu",
-            "i686-pc-windows-msvc",
-            "i686-unknown-linux-gnu",
-            "x86_64-unknown-linux-gnu",
-            "x86_64-apple-darwin",
-            "x86_64-pc-windows-gnu",
-            "x86_64-pc-windows-msvc",
-            # tier 2 with host tools
-            "aarch64-apple-darwin",
-            "aarch64-pc-windows-msvc",
-            "aarch64-unknown-linux-musl",
-            "arm-unknown-linux-gnueabi",
-            "arm-unknown-linux-gnueabihf",
-            "armv7-unknown-linux-gnueabihf",
-            "mips-unknown-linux-gnu",
-            "mips64-unknown-linux-gnuabi64",
-            "mips64el-unknown-linux-gnuabi64",
-            "mipsel-unknown-linux-gnu",
-            "powerpc-unknown-linux-gnu",
-            "powerpc64-unknown-linux-gnu",
-            "powerpc64le-unknown-linux-gnu",
-            "riscv64gc-unknown-linux-gnu",
-            "s390x-unknown-linux-gnu",
-            "x86_64-unknown-freebsd",
-            "x86_64-unknown-illumos",
-            "x86_64-unknown-linux-musl",
-            "x86_64-unknown-netbsd",
-        ]
-        return opt == "true" \
-            or (opt == "if-available" and self.build in supported_platforms)
-
     def _download_component_helper(
         self, filename, pattern, tarball_suffix, stage0=True, key=None
     ):
@@ -606,53 +531,6 @@ class RustBuild(object):
             )
         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 
-    def _download_ci_llvm(self, llvm_sha, llvm_assertions):
-        if not llvm_sha:
-            print("error: could not find commit hash for downloading LLVM")
-            print("help: maybe your repository history is too shallow?")
-            print("help: consider disabling `download-ci-llvm`")
-            print("help: or fetch enough history to include one upstream commit")
-            exit(1)
-        cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
-        cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, cache_prefix)
-        if not os.path.exists(rustc_cache):
-            os.makedirs(rustc_cache)
-
-        base = "https://ci-artifacts.rust-lang.org"
-        url = "rustc-builds/{}".format(llvm_sha)
-        if llvm_assertions:
-            url = url.replace('rustc-builds', 'rustc-builds-alt')
-        # ci-artifacts are only stored as .xz, not .gz
-        if not support_xz():
-            print("error: XZ support is required to download LLVM")
-            print("help: consider disabling `download-ci-llvm` or using python3")
-            exit(1)
-        tarball_suffix = '.tar.xz'
-        filename = "rust-dev-nightly-" + self.build + tarball_suffix
-        tarball = os.path.join(rustc_cache, filename)
-        if not os.path.exists(tarball):
-            help_on_error = "error: failed to download llvm from ci"
-            help_on_error += "\nhelp: old builds get deleted after a certain time"
-            help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
-            help_on_error += " disable `download-ci-llvm` in config.toml:"
-            help_on_error += "\n"
-            help_on_error += "\n[llvm]"
-            help_on_error += "\ndownload-ci-llvm = false"
-            help_on_error += "\n"
-            get(
-                base,
-                "{}/{}".format(url, filename),
-                tarball,
-                self.checksums_sha256,
-                verbose=self.verbose,
-                do_verify=False,
-                help_on_error=help_on_error,
-            )
-        unpack(tarball, tarball_suffix, self.llvm_root(),
-                match="rust-dev",
-                verbose=self.verbose)
-
     def fix_bin_or_dylib(self, fname):
         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
         or the RPATH section, to fix the dynamic library search path
@@ -816,17 +694,6 @@ class RustBuild(object):
         """
         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 
-    def llvm_stamp(self):
-        """Return the path for .llvm-stamp
-
-        >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
-        True
-        """
-        return os.path.join(self.llvm_root(), '.llvm-stamp')
-
-
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -856,22 +723,6 @@ class RustBuild(object):
             subdir = "ci-rustc"
         return os.path.join(self.build_dir, self.build, subdir)
 
-    def llvm_root(self):
-        """Return the CI LLVM root directory
-
-        >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
-        True
-
-        When the 'build' property is given should be a nested directory:
-
-        >>> rb.build = "devel"
-        >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
-        True
-        """
-        return os.path.join(self.build_dir, self.build, "ci-llvm")
-
     def get_toml(self, key, section=None):
         """Returns the value of the given key in config.toml, otherwise returns None
 

src/bootstrap/builder.rs

@@ -12,7 +12,6 @@ use std::process::Command;
 use std::time::{Duration, Instant};
 
 use crate::cache::{Cache, Interned, INTERNER};
-use crate::check;
 use crate::compile;
 use crate::config::{SplitDebuginfo, TargetSelection};
 use crate::dist;
@@ -25,6 +24,7 @@ use crate::test;
 use crate::tool::{self, SourceType};
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
 use crate::EXTRA_CHECK_CFGS;
+use crate::{check, Config};
 use crate::{Build, CLang, DocTests, GitRepo, Mode};
 
 pub use crate::Compiler;
@@ -960,6 +960,11 @@ impl<'a> Builder<'a> {
         None
     }
 
+    /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
+    pub(crate) fn llvm_link_shared(&self) -> bool {
+        Config::llvm_link_shared(self)
+    }
+
     /// Prepares an invocation of `cargo` to be run.
     ///
     /// This will create a `Command` that represents a pending execution of

src/bootstrap/compile.rs

@@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
             );
             cargo.env("LLVM_STATIC_STDCPP", file);
         }
-        if builder.config.llvm_link_shared {
+        if builder.llvm_link_shared() {
             cargo.env("LLVM_LINK_SHARED", "1");
         }
         if builder.config.llvm_use_libcxx {

src/bootstrap/config.rs

@@ -3,6 +3,7 @@
 //! This module implements parsing `config.toml` configuration files to tweak
 //! how the build runs.
 
+use std::cell::Cell;
 use std::cmp;
 use std::collections::{HashMap, HashSet};
 use std::env;
@@ -11,7 +12,7 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
-use crate::builder::TaskPath;
+use crate::builder::{Builder, TaskPath};
 use crate::cache::{Interned, INTERNER};
 use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
@@ -68,13 +69,14 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub llvm_libunwind: LlvmLibunwind,
     pub color: Color,
+    pub patch_binaries_for_nix: bool,
 
     pub on_fail: Option<String>,
     pub stage: u32,
     pub keep_stage: Vec<u32>,
     pub keep_stage_std: Vec<u32>,
     pub src: PathBuf,
-    // defaults to `config.toml`
+    /// defaults to `config.toml`
     pub config: PathBuf,
     pub jobs: Option<u32>,
     pub cmd: Subcommand,
@@ -95,7 +97,11 @@ pub struct Config {
     pub llvm_release_debuginfo: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
-    pub llvm_link_shared: bool,
+    /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
+    #[cfg(not(test))]
+    llvm_link_shared: Cell<Option<bool>>,
+    #[cfg(test)]
+    pub llvm_link_shared: Cell<Option<bool>>,
     pub llvm_clang_cl: Option<String>,
     pub llvm_targets: Option<String>,
     pub llvm_experimental_targets: Option<String>,
@@ -856,6 +862,7 @@ impl Config {
         set(&mut config.local_rebuild, build.local_rebuild);
         set(&mut config.print_step_timings, build.print_step_timings);
         set(&mut config.print_step_rusage, build.print_step_rusage);
+        set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
 
         config.verbose = cmp::max(config.verbose, flags.verbose);
 
@@ -911,7 +918,9 @@ impl Config {
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
-            set(&mut config.llvm_link_shared, llvm.link_shared);
+            if let Some(v) = llvm.link_shared {
+                config.llvm_link_shared.set(Some(v));
+            }
             config.llvm_targets = llvm.targets.clone();
             config.llvm_experimental_targets = llvm.experimental_targets.clone();
             config.llvm_link_jobs = llvm.link_jobs;
@@ -981,6 +990,7 @@ impl Config {
                 check_ci_llvm!(llvm.optimize);
                 check_ci_llvm!(llvm.thin_lto);
                 check_ci_llvm!(llvm.release_debuginfo);
+                // CI-built LLVM can be either dynamic or static. We won't know until we download it.
                 check_ci_llvm!(llvm.link_shared);
                 check_ci_llvm!(llvm.static_libstdcpp);
                 check_ci_llvm!(llvm.targets);
@@ -998,26 +1008,14 @@ impl Config {
                 check_ci_llvm!(llvm.clang);
                 check_ci_llvm!(llvm.build_config);
                 check_ci_llvm!(llvm.plugins);
-
-                // CI-built LLVM can be either dynamic or static.
-                let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
-                config.llvm_link_shared = if config.dry_run {
-                    // just assume dynamic for now
-                    true
-                } else {
-                    let link_type = t!(
-                        std::fs::read_to_string(ci_llvm.join("link-type.txt")),
-                        format!("CI llvm missing: {}", ci_llvm.display())
-                    );
-                    link_type == "dynamic"
-                };
             }
 
+            // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
             if config.llvm_thin_lto && llvm.link_shared.is_none() {
                 // If we're building with ThinLTO on, by default we want to link
                 // to LLVM shared, to avoid re-doing ThinLTO (which happens in
                 // the link step) with each stage.
-                config.llvm_link_shared = true;
+                config.llvm_link_shared.set(Some(true));
             }
         }
 
@@ -1272,6 +1270,42 @@ impl Config {
         }
     }
 
+    /// The absolute path to the downloaded LLVM artifacts.
+    pub(crate) fn ci_llvm_root(&self) -> PathBuf {
+        assert!(self.llvm_from_ci);
+        self.out.join(&*self.build.triple).join("ci-llvm")
+    }
+
+    /// Determine whether llvm should be linked dynamically.
+    ///
+    /// If `false`, llvm should be linked statically.
+    /// This is computed on demand since LLVM might have to first be downloaded from CI.
+    pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
+        let mut opt = builder.config.llvm_link_shared.get();
+        if opt.is_none() && builder.config.dry_run {
+            // just assume static for now - dynamic linking isn't supported on all platforms
+            return false;
+        }
+
+        let llvm_link_shared = *opt.get_or_insert_with(|| {
+            if builder.config.llvm_from_ci {
+                crate::native::maybe_download_ci_llvm(builder);
+                let ci_llvm = builder.config.ci_llvm_root();
+                let link_type = t!(
+                    std::fs::read_to_string(ci_llvm.join("link-type.txt")),
+                    format!("CI llvm missing: {}", ci_llvm.display())
+                );
+                link_type == "dynamic"
+            } else {
+                // unclear how thought-through this default is, but it maintains compatibility with
+                // previous behavior
+                false
+            }
+        });
+        builder.config.llvm_link_shared.set(opt);
+        llvm_link_shared
+    }
+
     pub fn verbose(&self) -> bool {
         self.verbose > 0
     }

src/bootstrap/dist.rs

@@ -1904,7 +1904,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
     // clear why this is the case, though. llvm-config will emit the versioned
     // paths and we don't want those in the sysroot (as we're expecting
     // unversioned paths).
-    if target.contains("apple-darwin") && builder.config.llvm_link_shared {
+    if target.contains("apple-darwin") && builder.llvm_link_shared() {
         let src_libdir = builder.llvm_out(target).join("lib");
         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
         if llvm_dylib_path.exists() {
@@ -1939,7 +1939,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection,
     // We do not need to copy LLVM files into the sysroot if it is not
     // dynamically linked; it is already included into librustc_llvm
     // statically.
-    if builder.config.llvm_link_shared {
+    if builder.llvm_link_shared() {
         maybe_install_llvm(builder, target, &dst_libdir);
     }
 }
@@ -1951,7 +1951,7 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection
     // We do not need to copy LLVM files into the sysroot if it is not
     // dynamically linked; it is already included into librustc_llvm
     // statically.
-    if builder.config.llvm_link_shared {
+    if builder.llvm_link_shared() {
         maybe_install_llvm(builder, target, &dst_libdir);
     }
 }
@@ -2077,7 +2077,7 @@ impl Step for RustDev {
         // compiler libraries.
         let dst_libdir = tarball.image_dir().join("lib");
         maybe_install_llvm(builder, target, &dst_libdir);
-        let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
+        let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
 
         Some(tarball.generate())

src/bootstrap/lib.rs

@@ -1391,6 +1391,23 @@ impl Build {
         paths
     }
 
+    pub fn rename(&self, src: &Path, dst: &Path) {
+        if self.config.dry_run {
+            return;
+        }
+        self.verbose_than(1, &format!("Move {:?} to {:?}", src, dst));
+        if src == dst {
+            return;
+        }
+        if let Err(e) = fs::rename(src, dst) {
+            if e.raw_os_error() == Some(libc::EXDEV) {
+                self.copy(src, dst);
+                return;
+            }
+            panic!("failed to rename `{}` to `{}`: {}", src.display(), dst.display(), e);
+        }
+    }
+
     /// Copies a file from `src` to `dst`
     pub fn copy(&self, src: &Path, dst: &Path) {
         if self.config.dry_run {

src/bootstrap/native.rs

@@ -12,9 +12,12 @@ use std::env;
 use std::env::consts::EXE_EXTENSION;
 use std::ffi::{OsStr, OsString};
 use std::fs::{self, File};
-use std::io;
+use std::io::{self, BufRead, BufReader, ErrorKind};
 use std::path::{Path, PathBuf};
-use std::process::Command;
+use std::process::{Command, Stdio};
+
+use once_cell::sync::OnceCell;
+use xz2::bufread::XzDecoder;
 
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::config::TargetSelection;
@@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config(
     builder: &Builder<'_>,
     target: TargetSelection,
 ) -> Result<PathBuf, Meta> {
+    maybe_download_ci_llvm(builder);
+
     // If we're using a custom LLVM bail out here, but we can only use a
     // custom LLVM for the build triple.
     if let Some(config) = builder.config.target_config.get(&target) {
@@ -111,6 +116,286 @@ pub fn prebuilt_llvm_config(
     Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
 }
 
+pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
+    let config = &builder.config;
+    if !config.llvm_from_ci {
+        return;
+    }
+    let mut rev_list = Command::new("git");
+    rev_list.args(&[
+        PathBuf::from("rev-list"),
+        "--author=bors@rust-lang.org".into(),
+        "-n1".into(),
+        "--first-parent".into(),
+        "HEAD".into(),
+        "--".into(),
+        builder.src.join("src/llvm-project"),
+        builder.src.join("src/bootstrap/download-ci-llvm-stamp"),
+        // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+        builder.src.join("src/version"),
+    ]);
+    let llvm_sha = output(&mut rev_list);
+    let llvm_sha = llvm_sha.trim();
+
+    if llvm_sha == "" {
+        println!("error: could not find commit hash for downloading LLVM");
+        println!("help: maybe your repository history is too shallow?");
+        println!("help: consider disabling `download-ci-llvm`");
+        println!("help: or fetch enough history to include one upstream commit");
+        panic!();
+    }
+
+    let llvm_root = config.ci_llvm_root();
+    let llvm_stamp = llvm_root.join(".llvm-stamp");
+    let key = format!("{}{}", llvm_sha, config.llvm_assertions);
+    if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
+        download_ci_llvm(builder, &llvm_sha);
+        for binary in ["llvm-config", "FileCheck"] {
+            fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary));
+        }
+        let llvm_lib = llvm_root.join("lib");
+        for entry in t!(fs::read_dir(&llvm_lib)) {
+            let lib = t!(entry).path();
+            if lib.ends_with(".so") {
+                fix_bin_or_dylib(builder, &lib);
+            }
+        }
+        t!(fs::write(llvm_stamp, key));
+    }
+}
+
+fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
+    let llvm_assertions = builder.config.llvm_assertions;
+
+    let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
+    let cache_dst = builder.out.join("cache");
+    let rustc_cache = cache_dst.join(cache_prefix);
+    if !rustc_cache.exists() {
+        t!(fs::create_dir_all(&rustc_cache));
+    }
+    let base = "https://ci-artifacts.rust-lang.org";
+    let url = if llvm_assertions {
+        format!("rustc-builds-alt/{}", llvm_sha)
+    } else {
+        format!("rustc-builds/{}", llvm_sha)
+    };
+    let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
+    let tarball = rustc_cache.join(&filename);
+    if !tarball.exists() {
+        download_component(builder, base, &format!("{}/{}", url, filename), &tarball);
+    }
+    let llvm_root = builder.config.ci_llvm_root();
+    unpack(builder, &tarball, &llvm_root);
+}
+
+/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
+/// or the RPATH section, to fix the dynamic library search path
+///
+/// This is only required on NixOS and uses the PatchELF utility to
+/// change the interpreter/RPATH of ELF executables.
+///
+/// Please see https://nixos.org/patchelf.html for more information
+fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) {
+    // FIXME: cache NixOS detection?
+    match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
+        Err(_) => return,
+        Ok(output) if !output.status.success() => return,
+        Ok(output) => {
+            let mut s = output.stdout;
+            if s.last() == Some(&b'\n') {
+                s.pop();
+            }
+            if s != b"Linux" {
+                return;
+            }
+        }
+    }
+
+    // If the user has asked binaries to be patched for Nix, then
+    // don't check for NixOS or `/lib`, just continue to the patching.
+    // FIXME: shouldn't this take precedence over the `uname` check above?
+    if !builder.config.patch_binaries_for_nix {
+        // Use `/etc/os-release` instead of `/etc/NIXOS`.
+        // The latter one does not exist on NixOS when using tmpfs as root.
+        let os_release = match File::open("/etc/os-release") {
+            Err(e) if e.kind() == ErrorKind::NotFound => return,
+            Err(e) => panic!("failed to access /etc/os-release: {}", e),
+            Ok(f) => f,
+        };
+        if !BufReader::new(os_release).lines().any(|l| t!(l).trim() == "ID=nixos") {
+            return;
+        }
+        if Path::new("/lib").exists() {
+            return;
+        }
+    }
+
+    // At this point we're pretty sure the user is running NixOS or using Nix
+    println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
+
+    // Only build `.nix-deps` once.
+    static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
+    let mut nix_build_succeeded = true;
+    let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
+        // Run `nix-build` to "build" each dependency (which will likely reuse
+        // the existing `/nix/store` copy, or at most download a pre-built copy).
+        //
+        // Importantly, we create a gc-root called `.nix-deps` in the `build/`
+        // directory, but still reference the actual `/nix/store` path in the rpath
+        // as it makes it significantly more robust against changes to the location of
+        // the `.nix-deps` location.
+        //
+        // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
+        // zlib: Needed as a system dependency of `libLLVM-*.so`.
+        // patchelf: Needed for patching ELF binaries (see doc comment above).
+        let nix_deps_dir = builder.out.join(".nix-deps");
+        const NIX_EXPR: &str = "
+        with (import <nixpkgs> {});
+        symlinkJoin {
+            name = \"rust-stage0-dependencies\";
+            paths = [
+                zlib
+                patchelf
+                stdenv.cc.bintools
+            ];
+        }
+        ";
+        nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[
+            Path::new("-E"),
+            Path::new(NIX_EXPR),
+            Path::new("-o"),
+            &nix_deps_dir,
+        ]));
+        nix_deps_dir
+    });
+    if !nix_build_succeeded {
+        return;
+    }
+
+    let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
+    let rpath_entries = {
+        // ORIGIN is a relative default, all binary and dynamic libraries we ship
+        // appear to have this (even when `../lib` is redundant).
+        // NOTE: there are only two paths here, delimited by a `:`
+        let mut entries = OsString::from("$ORIGIN/../lib:");
+        entries.push(t!(fs::canonicalize(nix_deps_dir)));
+        entries.push("/lib");
+        entries
+    };
+    patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
+    if !fname.ends_with(".so") {
+        // Finally, set the corret .interp for binaries
+        let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
+        // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
+        let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
+        patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
+    }
+
+    builder.try_run(patchelf.arg(fname));
+}
+
+fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) {
+    // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
+    let tempfile = t!(tempfile::NamedTempFile::new());
+    let temppath = tempfile.path().to_owned();
+    drop(tempfile);
+    let tempfile_str = temppath.to_str().expect("tempdir must be valid unicode");
+    // FIXME: support `do_verify` (only really needed for nightly rustfmt)
+    download_with_retries(builder, tempfile_str, &format!("{}/{}", base, url));
+    builder.rename(&temppath, dest_path);
+}
+
+fn download_with_retries(builder: &Builder<'_>, tempdir: &str, url: &str) {
+    println!("downloading {}", url);
+
+    // FIXME: check if curl is installed instead of skipping straight to powershell
+    if builder.build.build.contains("windows-msvc") {
+        for _ in 0..3 {
+            if builder.try_run(Command::new("PowerShell.exe").args(&[
+                "/nologo",
+                "-Command",
+                "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+                &format!(
+                    "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
+                    url, tempdir
+                ),
+            ])) {
+                return;
+            }
+            println!("\nspurious failure, trying again");
+        }
+    } else {
+        builder.run(Command::new("curl").args(&[
+            "-#",
+            "-y",
+            "30",
+            "-Y",
+            "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
+            "--connect-timeout",
+            "30", // timeout if cannot connect within 30 seconds
+            "--retry",
+            "3",
+            "-Sf",
+            "-o",
+            tempdir,
+            url,
+        ]));
+    }
+}
+
+fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) {
+    println!("extracting {} to {}", tarball.display(), dst.display());
+    if !dst.exists() {
+        t!(fs::create_dir_all(dst));
+    }
+
+    // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild
+    const MATCH: &str = "rust-dev";
+
+    // `tarball` ends with `.tar.xz`; strip that suffix
+    // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
+    let uncompressed_filename =
+        Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
+    let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
+
+    // decompress the file
+    let data = t!(File::open(tarball));
+    let decompressor = XzDecoder::new(BufReader::new(data));
+
+    let mut tar = tar::Archive::new(decompressor);
+    for member in t!(tar.entries()) {
+        let mut member = t!(member);
+        let original_path = t!(member.path()).into_owned();
+        // skip the top-level directory
+        if original_path == directory_prefix {
+            continue;
+        }
+        let mut short_path = t!(original_path.strip_prefix(directory_prefix));
+        if !short_path.starts_with(MATCH) {
+            continue;
+        }
+        short_path = t!(short_path.strip_prefix(MATCH));
+        let dst_path = dst.join(short_path);
+        builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
+        if !t!(member.unpack_in(dst)) {
+            panic!("path traversal attack ??");
+        }
+        let src_path = dst.join(original_path);
+        if src_path.is_dir() && dst_path.exists() {
+            continue;
+        }
+        t!(fs::rename(src_path, dst_path));
+    }
+    t!(fs::remove_dir_all(dst.join(directory_prefix)));
+}
+
+fn program_out_of_date(stamp: &Path, key: &str) -> bool {
+    if !stamp.exists() {
+        return true;
+    }
+    t!(fs::read_to_string(stamp)) != key
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Llvm {
     pub target: TargetSelection,
@@ -153,7 +438,7 @@ impl Step for Llvm {
             };
 
         builder.update_submodule(&Path::new("src").join("llvm-project"));
-        if builder.config.llvm_link_shared
+        if builder.llvm_link_shared()
             && (target.contains("windows") || target.contains("apple-darwin"))
         {
             panic!("shared linking to LLVM is not currently supported on {}", target.triple);
@@ -255,7 +540,7 @@ impl Step for Llvm {
         //
         // If we're not linking rustc to a dynamic LLVM, though, then don't link
         // tools to it.
-        if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared {
+        if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() {
             cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
         }